def prepare_stack(inputDir, filePattern, metadata=dict(), baseline_dict=dict(), update_mode=True): print('prepare .rsc file for ', filePattern) isce_files = sorted(glob.glob(os.path.join(os.path.abspath(inputDir), '*', filePattern))) if len(isce_files) == 0: raise FileNotFoundError('no file found in pattern: {}'.format(filePattern)) # write .rsc file for each interferogram file num_file = len(isce_files) prog_bar = ptime.progressBar(maxValue=num_file) for i in range(num_file): isce_file = isce_files[i] # prepare metadata for current file ifg_metadata = readfile.read_attribute(isce_file, metafile_ext='.xml') ifg_metadata.update(metadata) dates = os.path.basename(os.path.dirname(isce_file)).split('_') ifg_metadata = add_ifgram_metadata(ifg_metadata, dates, baseline_dict) # write .rsc file rsc_file = isce_file+'.rsc' writefile.write_roipac_rsc(ifg_metadata, rsc_file, update_mode=update_mode, print_msg=False) prog_bar.update(i+1, suffix='{}_{}'.format(dates[0], dates[1])) prog_bar.close() return
def run_2to3_timeseries(py2_file, py3_file): """Convert timeseries file from py2 pysar format to py3 pysar format""" # read data from py2_file atr = readfile.read_attribute(py2_file) length, width = int(atr['LENGTH']), int(atr['WIDTH']) with h5py.File(py2_file, 'r') as f: date_list = list(f['timeseries'].keys()) num_date = len(date_list) ts_data = np.zeros((num_date, length, width), np.float32) print('reading time-series ...') prog_bar = ptime.progressBar(maxValue=num_date) for i in range(num_date): ts_data[i, :, :] = f['timeseries/{}'.format(date_list[i])][:] prog_bar.update(i + 1, suffix=date_list[i]) prog_bar.close() # prepare metadata bperp = np.array([float(i) for i in atr['P_BASELINE_TIMESERIES'].split()], dtype=np.float32) dates = np.array(date_list, np.string_) atr['REF_DATE'] = date_list[0] for key in [ 'P_BASELINE_TIMESERIES', 'P_BASELINE_TOP_TIMESERIES', 'P_BASELINE_BOTTOM_TIMESERIES' ]: try: atr.pop(key) except: pass # write to py3_file ts_obj = timeseries(py3_file) ts_obj.write2hdf5(data=ts_data, dates=dates, bperp=bperp, metadata=atr) return py3_file
def bin_variance(distance, variance, step=5e3, min_pair_num=100e3, print_msg=True): x_steps = np.arange(0, np.max(distance), step) num_step = len(x_steps) std = np.zeros(x_steps.shape) stdStd = np.zeros(std.shape) p_num = np.zeros(x_steps.shape) if print_msg: prog_bar = ptime.progressBar(maxValue=num_step) for i in range(num_step): x = x_steps[i] idx = (distance > max(0, x - step / 2.)) * (distance < x + step / 2.) p_num[i] = np.sum(idx) std[i] = np.mean(np.sqrt(variance[idx])) stdStd[i] = np.std(np.sqrt(variance[idx])) if print_msg: prog_bar.update(i + 1, every=10) if print_msg: prog_bar.close() max_step_idx = int(max(np.argwhere(p_num > min_pair_num))) return x_steps[0:max_step_idx], std[0:max_step_idx], stdStd[0:max_step_idx]
def get_nonzero_phase_closure(ifgram_file, out_file=None, thres=0.1, unwDatasetName='unwrapPhase'): """Calculate/Read number of non-zero phase closure Parameters: ifgram_file : string, path of ifgram stack file out_file : string, path of num non-zero phase closure file Returns: num_nonzero_closure : 2D np.array in size of (length, width) """ if not out_file: out_file = 'numNonzeroPhaseClosure_{}.h5'.format(unwDatasetName) if os.path.isfile(out_file) and readfile.read_attribute(out_file): print('1. read number of nonzero phase closure from file: {}'.format( out_file)) num_nonzero_closure = readfile.read(out_file)[0] else: obj = ifgramStack(ifgram_file) obj.open(print_msg=False) length, width = obj.length, obj.width ref_phase = obj.get_reference_phase(unwDatasetName=unwDatasetName, dropIfgram=False) C = obj.get_design_matrix4triplet( obj.get_date12_list(dropIfgram=False)) # calculate phase closure line by line to save memory usage num_nonzero_closure = np.zeros((length, width), np.float32) print( '1. calculating phase closure of all pixels from dataset - {} ...'. format(unwDatasetName)) line_step = 10 num_loop = int(np.ceil(length / line_step)) prog_bar = ptime.progressBar(maxValue=num_loop) for i in range(num_loop): # read phase i0, i1 = i * line_step, min(length, (i + 1) * line_step) box = (0, i0, width, i1) pha_data = ifginv.read_unwrap_phase(obj, box, ref_phase, unwDatasetName=unwDatasetName, dropIfgram=False, print_msg=False) # calculate phase closure pha_closure = np.dot(C, pha_data) pha_closure = np.abs(pha_closure - ut.wrap(pha_closure)) # get number of non-zero phase closure num_nonzero = np.sum(pha_closure >= thres, axis=0) num_nonzero_closure[i0:i1, :] = num_nonzero.reshape(i1 - i0, width) prog_bar.update(i + 1, every=1, suffix='{}/{} lines'.format((i + 1) * line_step, length)) prog_bar.close() atr = dict(obj.metadata) atr['FILE_TYPE'] = 'mask' atr['UNIT'] = 1 writefile.write(num_nonzero_closure, out_file=out_file, metadata=atr) return num_nonzero_closure
def run_resample(self, src_data, interp_method='nearest', fill_value=np.nan, nprocs=1, print_msg=True): """Run interpolation operation for input 2D/3D data Parameters: src_data : 2D/3D np.array, source data to be geocoded interp_method : string, nearest | linear fill_value : NaN or number nprocs : int, number of processes to be used print_msg : bool Returns: geo_data : 2D/3D np.array """ # use pyresample if self.processor == 'pyresample': if len(src_data.shape) == 3: src_data = np.moveaxis(src_data, 0, -1) if src_data.dtype == np.bool_: fill_value = False print('restrict fill value to False for bool type source data') # resample source data into target data geo_data = self.run_pyresample(src_data=src_data, interp_method=interp_method, fill_value=fill_value, nprocs=nprocs, radius=None, print_msg=True) if len(geo_data.shape) == 3: geo_data = np.moveaxis(geo_data, -1, 0) # use scipy.interpolater.RegularGridInterpolator else: if print_msg: print( 'resampling using scipy.interpolate.RegularGridInterpolator ...' ) if len(src_data.shape) == 3: geo_data = np.empty( (src_data.shape[0], self.length, self.width), src_data.dtype) prog_bar = ptime.progressBar(maxValue=src_data.shape[0]) for i in range(src_data.shape[0]): geo_data[i, :, :] = self.run_regular_grid_interpolator( src_data=src_data[i, :, :], interp_method=interp_method, fill_value=fill_value, print_msg=True) prog_bar.update(i + 1) prog_bar.close() else: geo_data = self.run_regular_grid_interpolator( src_data=src_data, interp_method=interp_method, fill_value=fill_value, print_msg=True) return geo_data
def structure_function(data, lat, lon, step=5e3, min_pair_num=100e3, print_msg=True): num_sample = len(data) distance = np.zeros((num_sample**2)) variance = np.zeros((num_sample**2)) if print_msg: prog_bar = ptime.progressBar(maxValue=num_sample) for i in range(num_sample): distance[i * num_sample:(i + 1) * num_sample] = get_distance( lat, lon, i) variance[i * num_sample:(i + 1) * num_sample] = np.square(data - data[i]) if print_msg: prog_bar.update(i + 1, every=10) if print_msg: prog_bar.close() dist, std, stdStd = bin_variance(distance, variance, step=step, min_pair_num=min_pair_num, print_msg=print_msg) return dist, std, stdStd
def main(argv): try: file = argv[0] except: usage() sys.exit(1) g_name = 'unwrapPhase' g_name_out = 'unwrapPhase_laplace' print('calculate Laplace filter of {} based on approximate second derivatives.'.format(g_name)) f = h5py.File(file, 'a') ds = f[g_name] if g_name_out in f.keys(): ds_out = f[g_name_out] else: ds_out = f.create_dataset(g_name_out, shape=ds.shape, dtype=np.float32, chunks=True, compression=None) print('write to dataset /{}'.format(g_name_out)) num_ifgram = ds.shape[0] prog_bar = ptime.progressBar(maxValue=num_ifgram) for i in range(num_ifgram): unw = ds[i, :, :] ds_out[i, :, :] = laplace(unw) prog_bar.update(i+1, suffix='{}/{}'.format(i+1, num_ifgram)) prog_bar.close() f.close() print('finished writing to {}'.format(file)) return
def calculate_temporal_coherence_patch(ifgram_file, timeseries_file, box=None, ifg_num_file=None): atr = readfile.read_attribute(timeseries_file) if not box: box = (0, 0, int(atr['WIDTH']), int(atr['LENGTH'])) # Read timeseries data ts_obj = timeseries(timeseries_file) ts_obj.open(print_msg=False) print('reading timeseries data from file: {}'.format(timeseries_file)) ts_data = ts_obj.read(box=box, print_msg=False).reshape(ts_obj.numDate, -1) ts_data = ts_data[1:, :] ts_data *= -4 * np.pi / float(atr['WAVELENGTH']) # Read ifgram data stack_obj = ifgramStack(ifgram_file) stack_obj.open(print_msg=False) A = stack_obj.get_design_matrix4timeseries_estimation(dropIfgram=True)[0] print('reading unwrapPhase data from file: {}'.format(ifgram_file)) ifgram_data = stack_obj.read(datasetName='unwrapPhase', box=box).reshape(A.shape[0], -1) ref_value = stack_obj.get_reference_phase(dropIfgram=True).reshape((-1, 1)) ifgram_data -= np.tile(ref_value, (1, ifgram_data.shape[1])) ifgram_diff = ifgram_data - np.dot(A, ts_data) del ts_data pixel_num = ifgram_data.shape[1] temp_coh = np.zeros((pixel_num), np.float32) # (fast) nasty solution, which used all phase value including invalid zero phase if not ifg_num_file: temp_coh = np.abs(np.sum(np.exp(1j * ifgram_diff), axis=0)) / ifgram_diff.shape[0] # (slow) same solution as ifgram_inversion.py, considering: # 1) invalid zero phase in ifgram # 2) design matrix rank deficiency. else: print( 'considering different number of interferograms used in network inversion for each pixel' ) ifg_num_map = readfile.read(ifg_num_file, box=box)[0].flatten() prog_bar = ptime.progressBar(maxValue=pixel_num) for i in range(pixel_num): if ifg_num_map[i] > 0: idx = ifgram_data[:, i] != 0. temp_diff = ifgram_diff[idx, i] temp_coh[i] = np.abs(np.sum(np.exp(1j * temp_diff), axis=0)) / temp_diff.shape[0] prog_bar.update(i + 1, every=1000, suffix='{}/{}'.format(i + 1, pixel_num)) prog_bar.close() temp_coh = np.reshape(temp_coh, (box[3] - box[1], box[2] - box[0])) return temp_coh
def main(argv): try: file = argv[0] except: usage() sys.exit(1) outfile = os.path.splitext(file)[0] + '_wrap' + os.path.splitext(file)[1] one_cycle = 2 * np.pi one_cycle = 0.05 atr = readfile.read_attribute(file) k = atr['FILE_TYPE'] if k in ['interferograms', 'coherence', 'wrapped', 'timeseries']: h5 = h5py.File(file, 'r') epochList = sorted(h5[k].keys()) epoch_num = len(epochList) prog_bar = ptime.progressBar(maxValue=epoch_num) print('writing >>> ' + outfile) h5out = h5py.File(outfile, 'w') group = h5out.create_group(k) if k in ['interferograms', 'coherence', 'wrapped']: date12_list = ptime.list_ifgram2date12(epochList) print('number of interferograms: ' + str(len(epochList))) for i in range(epoch_num): epoch = epochList[i] data = h5[k][epoch].get(epoch)[:] data_wrap = rewrap(data) gg = group.create_group(epoch) dset = gg.create_dataset(epoch, data=data_wrap) for key, value in h5[k][epoch].attrs.items(): gg.attrs[key] = value prog_bar.update(i + 1, suffix=date12_list[i]) elif k == 'timeseries': print('number of acquisitions: ' + str(len(epochList))) for i in range(epoch_num): epoch = epochList[i] data = h5[k].get(epoch)[:] data_wrap = rewrap(data, one_cycle) dset = group.create_dataset(epoch, data=data_wrap) prog_bar.update(i + 1, suffix=epoch) for key, value in h5[k].attrs.items(): group.attrs[key] = value h5.close() h5out.close() prog_bar.close() print('Done.') return outfile
def main(iargs=None): inps = cmd_line_parse(iargs) # read timeseries info / data obj = timeseries(inps.timeseries_file) obj.open() tbase = np.array(obj.yearList, np.float32).reshape(-1, 1) tbase -= tbase[obj.refIndex] ts_data = obj.read().reshape(obj.numDate, -1) # Smooth acquisitions / moving window in time one by one print('-' * 50) print('filtering in time Gaussian window with size of {:.1f} years'.format( inps.time_win)) ts_data_filt = np.zeros(ts_data.shape, np.float32) prog_bar = ptime.progressBar(maxValue=obj.numDate) for i in range(obj.numDate): # Weight from Gaussian (normal) distribution in time tbase_diff = tbase[i] - tbase weight = np.exp(-0.5 * (tbase_diff**2) / (inps.time_win**2)) weight /= np.sum(weight) # Smooth the current acquisition ts_data_filt[i, :] = np.sum(ts_data * weight, axis=0) prog_bar.update(i + 1, suffix=obj.dateList[i]) prog_bar.close() del ts_data ts_data_filt -= ts_data_filt[obj.refIndex, :] ts_data_filt = np.reshape(ts_data_filt, (obj.numDate, obj.length, obj.width)) # write filtered timeseries file if not inps.outfile: inps.outfile = '{}_tempGaussian.h5'.format( os.path.splitext(inps.timeseries_file)[0]) obj_out = timeseries(inps.outfile) obj_out.write2hdf5(ts_data_filt, refFile=inps.timeseries_file) return inps.outfile
def run_unwrap_error_bridge(ifgram_file, water_mask_file, ramp_type=None, radius=50, ccName='connectComponent', dsNameIn='unwrapPhase', dsNameOut='unwrapPhase_bridging'): """Run unwrapping error correction with bridging Parameters: ifgram_file : str, path of ifgram stack file water_mask_file : str, path of water mask file ramp_type : str, name of phase ramp to be removed during the phase jump estimation ccName : str, dataset name of connected components dsNameIn : str, dataset name of unwrap phase to be corrected dsNameOut : str, dataset name of unwrap phase to be saved after correction Returns: ifgram_file : str, path of ifgram stack file """ print('-' * 50) print( 'correct unwrapping error in {} with bridging ...'.format(ifgram_file)) if ramp_type is not None: print('estimate and remove a {} ramp while calculating phase offset'. format(ramp_type)) # read water mask if water_mask_file and os.path.isfile(water_mask_file): print('read water mask from file:', water_mask_file) water_mask = readfile.read(water_mask_file)[0] else: water_mask = None # file info atr = readfile.read_attribute(ifgram_file) length, width = int(atr['LENGTH']), int(atr['WIDTH']) k = atr['FILE_TYPE'] # correct unwrap error ifgram by ifgram if k == 'ifgramStack': date12_list = ifgramStack(ifgram_file).get_date12_list( dropIfgram=False) num_ifgram = len(date12_list) shape_out = (num_ifgram, length, width) # prepare output data writing print('open {} with r+ mode'.format(ifgram_file)) f = h5py.File(ifgram_file, 'r+') print('input dataset:', dsNameIn) print('output dataset:', dsNameOut) if dsNameOut in f.keys(): ds = f[dsNameOut] print('access /{d} of np.float32 in size of {s}'.format( d=dsNameOut, s=shape_out)) else: ds = f.create_dataset(dsNameOut, shape_out, maxshape=(None, None, None), chunks=True, compression=None) print('create /{d} of np.float32 in size of {s}'.format( d=dsNameOut, s=shape_out)) # correct unwrap error ifgram by ifgram prog_bar = ptime.progressBar(maxValue=num_ifgram) for i in range(num_ifgram): # read unwrapPhase and connectComponent date12 = date12_list[i] unw = np.squeeze(f[dsNameIn][i, :, :]) cc = np.squeeze(f[ccName][i, :, :]) if water_mask is not None: cc[water_mask == 0] = 0 # bridging cc_obj = connectComponent(conncomp=cc, metadata=atr) cc_obj.label() cc_obj.find_mst_bridge() unw_cor = cc_obj.unwrap_conn_comp(unw, ramp_type=ramp_type) # write to hdf5 file ds[i, :, :] = unw_cor prog_bar.update(i + 1, suffix=date12) prog_bar.close() ds.attrs['MODIFICATION_TIME'] = str(time.time()) f.close() print('close {} file.'.format(ifgram_file)) if k == '.unw': # read unwrap phase unw = readfile.read(ifgram_file)[0] # read connected components cc_files0 = [ ifgram_file + '.conncomp', os.path.splitext(ifgram_file)[0] + '_snap_connect.byt' ] cc_files = [i for i in cc_files0 if os.path.isfile(i)] if len(cc_files) == 0: raise FileNotFoundError(cc_files0) cc = readfile.read(cc_files[0])[0] if water_mask is not None: cc[water_mask == 0] = 0 # bridging cc_obj = connectComponent(conncomp=cc, metadata=atr) cc_obj.label() cc_obj.find_mst_bridge() unw_cor = cc_obj.unwrap_conn_comp(unw, ramp_type=ramp_type) # write to hdf5 file out_file = '{}_unwCor{}'.format( os.path.splitext(ifgram_file)[0], os.path.splitext(ifgram_file)[1]) print('writing >>> {}'.format(out_file)) writefile.write(unw_cor, out_file=out_file, ref_file=ifgram_file) return ifgram_file
def get_delay_timeseries(inps, atr): """Calculate delay time-series and write it to HDF5 file. Parameters: inps : namespace, all input parameters atr : dict, metadata to be saved in trop_file Returns: trop_file : str, file name of ECMWF.h5 """ def get_dataset_size(fname): atr = readfile.read_attribute(fname) return (atr['LENGTH'], atr['WIDTH']) # check 1 - existing tropo delay file if (ut.run_or_skip(out_file=inps.trop_file, in_file=inps.grib_file_list, print_msg=False) == 'skip' and get_dataset_size(inps.trop_file) == get_dataset_size( inps.geom_file)): print( '{} file exists and is newer than all GRIB files, skip updating.'. format(inps.trop_file)) return # check 2 - geometry file if any(i is None for i in [inps.geom_file, inps.ref_yx]): print( 'No DEM / incidenceAngle / ref_yx found, skip calculating tropospheric delays.' ) if not os.path.isfile(inps.trop_file): inps.trop_file = None return # prepare geometry data geom_obj = geometry(inps.geom_file) geom_obj.open() inps.dem = geom_obj.read(datasetName='height') inps.inc = geom_obj.read(datasetName='incidenceAngle') if 'latitude' in geom_obj.datasetNames: inps.lat = geom_obj.read(datasetName='latitude') inps.lon = geom_obj.read(datasetName='longitude') else: inps.lat, inps.lon = get_lat_lon(geom_obj.metadata) # calculate phase delay length, width = int(atr['LENGTH']), int(atr['WIDTH']) num_date = len(inps.grib_file_list) date_list = [str(re.findall('\d{8}', i)[0]) for i in inps.grib_file_list] trop_data = np.zeros((num_date, length, width), np.float32) print( 'calcualting delay for each date using PyAPS (Jolivet et al., 2011; 2014) ...' ) print('number of grib files used: {}'.format(num_date)) if verbose: prog_bar = ptime.progressBar(maxValue=num_date) for i in range(num_date): grib_file = inps.grib_file_list[i] trop_data[i] = get_delay(grib_file, inps) if verbose: prog_bar.update(i + 1, suffix=os.path.basename(grib_file)) if verbose: prog_bar.close() # Convert relative phase delay on reference date inps.ref_date = atr.get('REF_DATE', date_list[0]) print('convert to relative phase delay with reference date: ' + inps.ref_date) inps.ref_idx = date_list.index(inps.ref_date) trop_data -= np.tile(trop_data[inps.ref_idx, :, :], (num_date, 1, 1)) # Write tropospheric delay to HDF5 atr['REF_Y'] = inps.ref_yx[0] atr['REF_X'] = inps.ref_yx[1] ts_obj = timeseries(inps.trop_file) ts_obj.write2hdf5(data=trop_data, dates=date_list, metadata=atr, refFile=inps.timeseries_file) return
def run_unwrap_error_bridge(ifgram_file, mask_cc_file, bridges, dsNameIn='unwrapPhase', dsNameOut='unwrapPhase_bridging', ramp_type=None): """Run unwrapping error correction with bridging Parameters: ifgram_file : str, path of ifgram stack file mask_cc_file : str, path of conn comp mask file bridges : list of dicts, check bridge_unwrap_error() for details dsNameIn : str, dataset name of unwrap phase to be corrected dsNameOut : str, dataset name of unwrap phase to be saved after correction ramp_type : str, name of phase ramp to be removed during the phase jump estimation Returns: ifgram_file : str, path of ifgram stack file """ print('-'*50) print('correct unwrapping error in {} with bridging ...'.format(ifgram_file)) # file info atr = readfile.read_attribute(ifgram_file) length, width = int(atr['LENGTH']), int(atr['WIDTH']) ref_y, ref_x = int(atr['REF_Y']), int(atr['REF_X']) k = atr['FILE_TYPE'] # read mask print('read mask from file: {}'.format(mask_cc_file)) mask_cc = readfile.read(mask_cc_file, datasetName='mask')[0] if ramp_type is not None: print('estimate and remove phase ramp of {} during the correction'.format(ramp_type)) mask4ramp = (mask_cc == mask_cc[ref_y, ref_x]) # correct unwrap error ifgram by ifgram if k == 'ifgramStack': date12_list = ifgramStack(ifgram_file).get_date12_list(dropIfgram=False) num_ifgram = len(date12_list) shape_out = (num_ifgram, length, width) # prepare output data writing print('open {} with r+ mode'.format(ifgram_file)) f = h5py.File(ifgram_file, 'r+') print('input dataset:', dsNameIn) print('output dataset:', dsNameOut) if dsNameOut in f.keys(): ds = f[dsNameOut] print('access /{d} of np.float32 in size of {s}'.format(d=dsNameOut, s=shape_out)) else: ds = f.create_dataset(dsNameOut, shape_out, maxshape=(None, None, None), chunks=True, compression=None) print('create /{d} of np.float32 in size of {s}'.format(d=dsNameOut, s=shape_out)) # correct unwrap error ifgram by ifgram prog_bar = ptime.progressBar(maxValue=num_ifgram) for i in range(num_ifgram): # read unwrapPhase date12 = date12_list[i] unw = np.squeeze(f[dsNameIn][i, :, :]) unw[unw != 0.] -= unw[ref_y, ref_x] # remove phase ramp before phase jump estimation if ramp_type is not None: unw, unw_ramp = deramp(unw, mask4ramp, ramp_type, metadata=atr) # estimate/correct phase jump unw_cor = bridge_unwrap_error(unw, mask_cc, bridges) if ramp_type is not None: unw_cor += unw_ramp # write to hdf5 file ds[i, :, :] = unw_cor prog_bar.update(i+1, suffix=date12) prog_bar.close() ds.attrs['MODIFICATION_TIME'] = str(time.time()) f.close() print('close {} file.'.format(ifgram_file)) if k == '.unw': # read data unw = readfile.read(ifgram_file)[0] unw[unw != 0.] -= unw[ref_y, ref_x] # remove phase ramp before phase jump estimation if ramp_type is not None: unw, unw_ramp = deramp(unw, mask4ramp, ramp_type, metadata=atr) # estimate/correct phase jump unw_cor = bridge_unwrap_error(unw, mask_cc, bridges) if ramp_type is not None: unw_cor += unw_ramp # write to hdf5 file out_file = '{}_unwCor{}'.format(os.path.splitext(ifgram_file)[0], os.path.splitext(ifgram_file)[1]) print('writing >>> {}'.format(out_file)) writefile.write(unw_cor, out_file=out_file, ref_file=ifgram_file) return ifgram_file
def run_unwrap_error_patch(ifgram_file, box=None, mask_file=None, ref_phase=None, fast_mode=False, thres=0.1, dsNameIn='unwrapPhase'): """Estimate/Correct unwrapping error in ifgram stack on area defined by box. Parameters: ifgram_file : string, ifgramStack file box : tuple of 4 int, indicating areas to be read and analyzed mask_file : string, file name of mask file for pixels to be analyzed ref_pahse : 1D np.array in size of (num_ifgram,) phase value on reference pixel, because: 1) phase value stored in pysar is not reference yet 2) reference point may be out of box definition fast_mode : bool, apply zero jump constraint on ifgrams without unwrapping error. thres : float, threshold of non-zero phase closure to be identified as unwrapping error. Returns: pha_data : 3D np.array in size of (num_ifgram_all, box[3]-box[2], box[2]-box[0]), unwrapped phase value after error correction """ # Basic info stack_obj = ifgramStack(ifgram_file) stack_obj.open(print_msg=False) num_ifgram = stack_obj.numIfgram # Size Info - Patch if box: num_row = box[3] - box[1] num_col = box[2] - box[0] else: num_row = stack_obj.length num_col = stack_obj.width num_pixel = num_row * num_col C = stack_obj.get_design_matrix4ifgram_triangle(dropIfgram=True) print('number of interferograms: {}'.format(C.shape[1])) print('number of triangles: {}'.format(C.shape[0])) # read unwrapPhase pha_data_all = ifginv.read_unwrap_phase(stack_obj, box, ref_phase, unwDatasetName=dsNameIn, dropIfgram=False) pha_data = np.array(pha_data_all[stack_obj.dropIfgram, :]) # mask of pixels to analyze mask = np.ones((num_pixel), np.bool_) print('number of pixels read: {}'.format(num_pixel)) # mask 1. mask of water or area of interest if mask_file: dsNames = readfile.get_dataset_list(mask_file) dsName = [i for i in dsNames if i in ['waterMask', 'mask']][0] waterMask = readfile.read(mask_file, datasetName=dsName, box=box)[0].flatten() mask *= np.array(waterMask, np.bool_) del waterMask print('number of pixels left after mask: {}'.format(np.sum(mask))) # mask 2. mask of pixels without unwrap error: : zero phase closure on all triangles print('calculating phase closure of all possible triangles ...') pha_closure = np.dot(C, pha_data) pha_closure = np.abs(pha_closure - ut.wrap(pha_closure)) # Eq 4.2 (Fattahi, 2015) num_nonzero_closure = np.sum(pha_closure >= thres, axis=0) mask *= (num_nonzero_closure != 0.) del pha_closure print('number of pixels left after checking phase closure: {}'.format(np.sum(mask))) # mask summary num_pixel2proc = int(np.sum(mask)) if num_pixel2proc > 0: ifgram = pha_data[:, mask] ifgram_cor = np.array(ifgram, np.float32) print('number of pixels to process: {} out of {} ({:.2f}%)'.format(num_pixel2proc, num_pixel, num_pixel2proc/num_pixel*100)) # correcting unwrap error based on phase closure print('correcting unwrapping error ...') if fast_mode: ifgram_cor = correct_unwrap_error(ifgram, C, Dconstraint=False)[0] else: prog_bar = ptime.progressBar(maxValue=num_pixel2proc) for i in range(num_pixel2proc): ifgram_cor[:, i] = correct_unwrap_error(ifgram[:, i], C, Dconstraint=True)[0].flatten() prog_bar.update(i+1, every=10, suffix='{}/{}'.format(i+1, num_pixel2proc)) prog_bar.close() pha_data[:, mask] = ifgram_cor pha_data_all[stack_obj.dropIfgram, :] = pha_data pha_data_all = pha_data_all.reshape(num_ifgram, num_row, num_col) num_nonzero_closure = num_nonzero_closure.reshape(num_row, num_col) return pha_data_all, num_nonzero_closure
def get_common_region_int_ambiguity(ifgram_file, cc_mask_file, water_mask_file=None, num_sample=100, dsNameIn='unwrapPhase'): """Solve the phase unwrapping integer ambiguity for the common regions among all interferograms Parameters: ifgram_file : str, path of interferogram stack file cc_mask_file : str, path of common connected components file water_mask_file : str, path of water mask file num_sample : int, number of pixel sampled for each region dsNameIn : str, dataset name of the unwrap phase to be corrected Returns: common_regions : list of skimage.measure._regionprops._RegionProperties object modified by adding two more variables: sample_coords : 2D np.ndarray in size of (num_sample, 2) in int64 format int_ambiguity : 1D np.ndarray in size of (num_ifgram,) in int format """ print('-' * 50) print( 'calculating the integer ambiguity for the common regions defined in', cc_mask_file) # stack info stack_obj = ifgramStack(ifgram_file) stack_obj.open() date12_list = stack_obj.get_date12_list(dropIfgram=True) num_ifgram = len(date12_list) C = matrix( ifgramStack.get_design_matrix4triplet(date12_list).astype(float)) ref_phase = stack_obj.get_reference_phase(unwDatasetName=dsNameIn, dropIfgram=True).reshape( num_ifgram, -1) # prepare common label print('read common mask from', cc_mask_file) cc_mask = readfile.read(cc_mask_file)[0] if water_mask_file is not None and os.path.isfile(water_mask_file): water_mask = readfile.read(water_mask_file)[0] print('refine common mask based on water mask file', water_mask_file) cc_mask[water_mask == 0] = 0 label_img, num_label = connectComponent.get_large_label(cc_mask, min_area=2.5e3, print_msg=True) common_regions = measure.regionprops(label_img) print('number of common regions:', num_label) # add sample_coords / int_ambiguity print('number of samples per region:', num_sample) print('solving the phase-unwrapping integer ambiguity for {}'.format( dsNameIn)) print( '\tbased on the closure phase of interferograms triplets (Yunjun et al., 2019)' ) print( '\tusing the L1-norm regularzed least squares approximation (LASSO) ...' ) for i in range(num_label): common_reg = common_regions[i] # sample_coords idx = sorted( np.random.choice(common_reg.area, num_sample, replace=False)) common_reg.sample_coords = common_reg.coords[idx, :].astype(int) # solve for int_ambiguity U = np.zeros((num_ifgram, num_sample)) if common_reg.label == label_img[stack_obj.refY, stack_obj.refX]: print('{}/{} skip calculation for the reference region'.format( i + 1, num_label)) else: prog_bar = ptime.progressBar(maxValue=num_sample, prefix='{}/{}'.format( i + 1, num_label)) for j in range(num_sample): # read unwrap phase y, x = common_reg.sample_coords[j, :] unw = ifginv.read_unwrap_phase(stack_obj, box=(x, y, x + 1, y + 1), ref_phase=ref_phase, unwDatasetName=dsNameIn, dropIfgram=True, print_msg=False).reshape( num_ifgram, -1) # calculate closure_int closure_pha = np.dot(C, unw) closure_int = matrix( np.round( (closure_pha - ut.wrap(closure_pha)) / (2. * np.pi))) # solve for U U[:, j] = np.round( l1regls(-C, closure_int, alpha=1e-2, show_progress=0)).flatten() prog_bar.update(j + 1, every=5) prog_bar.close() # add int_ambiguity common_reg.int_ambiguity = np.median(U, axis=1) common_reg.date12_list = date12_list #sort regions by size to facilitate the region matching later common_regions.sort(key=lambda x: x.area, reverse=True) # plot sample result fig_size = pp.auto_figure_size(label_img.shape, disp_cbar=False) fig, ax = plt.subplots(figsize=fig_size) ax.imshow(label_img, cmap='jet') for common_reg in common_regions: ax.plot(common_reg.sample_coords[:, 1], common_reg.sample_coords[:, 0], 'k.', ms=2) pp.auto_flip_direction(stack_obj.metadata, ax, print_msg=False) out_img = 'common_region_sample.png' fig.savefig(out_img, bbox_inches='tight', transparent=True, dpi=300) print('saved common regions and sample pixels to file', out_img) return common_regions
def write2hdf5(self, outputFile='geometryRadar.h5', access_mode='w', box=None, compression='gzip', extra_metadata=None): ''' / Root level Attributes Dictionary for metadata. 'X/Y_FIRST/STEP' attribute for geocoded. /height 2D array of float32 in size of (l, w ) in meter. /latitude (azimuthCoord) 2D array of float32 in size of (l, w ) in degree. /longitude (rangeCoord) 2D array of float32 in size of (l, w ) in degree. /incidenceAngle 2D array of float32 in size of (l, w ) in degree. /slantRangeDistance 2D array of float32 in size of (l, w ) in meter. /azimuthAngle 2D array of float32 in size of (l, w ) in degree. (optional) /shadowMask 2D array of bool in size of (l, w ). (optional) /waterMask 2D array of bool in size of (l, w ). (optional) /bperp 3D array of float32 in size of (n, l, w) in meter (optional) /date 1D array of string in size of (n, ) in YYYYMMDD(optional) ... ''' if len(self.datasetDict) == 0: print( 'No dataset file path in the object, skip HDF5 file writing.') return None self.outputFile = outputFile f = h5py.File(self.outputFile, access_mode) print('create HDF5 file {} with {} mode'.format( self.outputFile, access_mode)) #groupName = self.name #group = f.create_group(groupName) #print('create group /{}'.format(groupName)) maxDigit = max([len(i) for i in geometryDatasetNames]) length, width = self.get_size(box=box) self.length, self.width = self.get_size() ############################### for dsName in self.dsNames: # 3D datasets containing bperp if dsName == 'bperp': self.dateList = list(self.datasetDict[dsName].keys()) dsDataType = dataType self.numDate = len(self.dateList) dsShape = (self.numDate, length, width) ds = f.create_dataset(dsName, shape=dsShape, maxshape=(None, dsShape[1], dsShape[2]), dtype=dsDataType, chunks=True, compression=compression) print(('create dataset /{d:<{w}} of {t:<25} in size of {s}' ' with compression = {c}').format(d=dsName, w=maxDigit, t=str(dsDataType), s=dsShape, c=str(compression))) print( 'read coarse grid baseline files and linear interpolate into full resolution ...' ) prog_bar = ptime.progressBar(maxValue=self.numDate) for i in range(self.numDate): fname = self.datasetDict[dsName][self.dateList[i]] data = read_isce_bperp_file(fname=fname, out_shape=(self.length, self.width), box=box) ds[i, :, :] = data prog_bar.update(i + 1, suffix=self.dateList[i]) prog_bar.close() # Write 1D dataset date dsName = 'date' dsShape = (self.numDate, ) dsDataType = np.string_ print(('create dataset /{d:<{w}} of {t:<25}' ' in size of {s}').format(d=dsName, w=maxDigit, t=str(dsDataType), s=dsShape)) data = np.array(self.dateList, dtype=dsDataType) ds = f.create_dataset(dsName, data=data) # 2D datasets containing height, latitude, incidenceAngle, shadowMask, etc. else: dsDataType = dataType if dsName.lower().endswith('mask'): dsDataType = np.bool_ dsShape = (length, width) print(('create dataset /{d:<{w}} of {t:<25} in size of {s}' ' with compression = {c}').format(d=dsName, w=maxDigit, t=str(dsDataType), s=dsShape, c=str(compression))) data = np.array(self.read(family=dsName, box=box)[0], dtype=dsDataType) ds = f.create_dataset(dsName, data=data, chunks=(100, 300), compression=compression) ############################### # Generate Dataset if not existed in binary file: incidenceAngle, slantRangeDistance for dsName in [ i for i in ['incidenceAngle', 'slantRangeDistance'] if i not in self.dsNames ]: # Calculate data data = None if dsName == 'incidenceAngle': data = self.get_incidence_angle(box=box) elif dsName == 'slantRangeDistance': data = self.get_slant_range_distance(box=box) # Write dataset if data is not None: dsShape = data.shape dsDataType = dataType print(('create dataset /{d:<{w}} of {t:<25} in size of {s}' ' with compression = {c}').format(d=dsName, w=maxDigit, t=str(dsDataType), s=dsShape, c=str(compression))) ds = f.create_dataset(dsName, data=data, dtype=dataType, chunks=(100, 300), compression=compression) ############################### # Attributes self.get_metadata() if extra_metadata: self.metadata.update(extra_metadata) print('add extra metadata: {}'.format(extra_metadata)) self.metadata = ut.subset_attribute(self.metadata, box) self.metadata['FILE_TYPE'] = self.name for key, value in self.metadata.items(): f.attrs[key] = value f.close() print('Finished writing to {}'.format(self.outputFile)) return self.outputFile
def get_delay_timeseries(inps, atr): """Calculate delay time-series and write it to HDF5 file. Parameters: inps : namespace, all input parameters atr : dict, metadata to be saved in trop_file Returns: trop_file : str, file name of ECMWF.h5 """ def get_dataset_size(fname): atr = readfile.read_attribute(fname) return (atr['LENGTH'], atr['WIDTH']) if (ut.run_or_skip(out_file=inps.trop_file, in_file=inps.grib_file_list, print_msg=False) == 'skip' and get_dataset_size(inps.trop_file) == get_dataset_size( inps.geom_file)): print( '{} file exists and is newer than all GRIB files, skip updating.'. format(inps.trop_file)) else: if any(i is None for i in [inps.geom_file, inps.ref_yx]): print( 'No DEM / incidenceAngle / ref_yx found, skip calculating tropospheric delays.' ) if not os.path.isfile(inps.trop_file): inps.trop_file = None return # calculate phase delay length, width = int(atr['LENGTH']), int(atr['WIDTH']) num_date = len(inps.grib_file_list) date_list = [ str(re.findall('\d{8}', i)[0]) for i in inps.grib_file_list ] trop_data = np.zeros((num_date, length, width), np.float32) print( 'calcualting delay for each date using PyAPS (Jolivet et al., 2011; 2014) ...' ) print('number of grib files used: {}'.format(num_date)) prog_bar = ptime.progressBar(maxValue=num_date) for i in range(num_date): grib_file = inps.grib_file_list[i] trop_data[i] = get_delay(grib_file, inps) prog_bar.update(i + 1, suffix=os.path.basename(grib_file)) prog_bar.close() # Convert relative phase delay on reference date try: inps.ref_date = atr['REF_DATE'] except: inps.ref_date = date_list[0] print('convert to relative phase delay with reference date: ' + inps.ref_date) inps.ref_idx = date_list.index(inps.ref_date) trop_data -= np.tile(trop_data[inps.ref_idx, :, :], (num_date, 1, 1)) # Write tropospheric delay to HDF5 atr['REF_Y'] = inps.ref_yx[0] atr['REF_X'] = inps.ref_yx[1] ts_obj = timeseries(inps.trop_file) ts_obj.write2hdf5(data=trop_data, dates=date_list, metadata=atr, refFile=inps.timeseries_file) # Delete temporary DEM file in ROI_PAC format if inps.geom_file: temp_files = [ fname for fname in [inps.dem_file, inps.inc_angle_file, inps.lat_file, inps.lon_file] if (fname is not None and 'pyaps' in fname) ] if temp_files: print('delete temporary geometry files') rmCmd = 'rm ' for fname in temp_files: rmCmd += ' {f} {f}.rsc '.format(f=fname) print(rmCmd) os.system(rmCmd) return
def ifgram_inversion_patch(ifgram_file, box=None, ref_phase=None, weight_func='fim', mask_dataset_name=None, mask_threshold=0.4, water_mask_file=None, skip_zero_phase=True): """Invert one patch of an ifgram stack into timeseries. Parameters: ifgram_file : str, interferograms stack HDF5 file, e.g. ./INPUTS/ifgramStack.h5 box : tuple of 4 int, indicating (x0, y0, x1, y1) pixel coordinate of area of interest or None, to process the whole file and write output file ref_phase : 1D array in size of (num_ifgram) or None weight_func : str, weight function, choose in ['sbas', 'fim', 'var', 'coh'] mask_dataset_name : str, dataset name in ifgram_file used to mask unwrapPhase pixelwisely mask_threshold : float, min coherence of pixels if mask_dataset_name='coherence' water_mask_file : str, water mask filename if available, skip inversion on water to speed up the process skip_zero_phase : bool, skip zero value of unwrapped phase or not, default yes, for comparison Returns: ts : 3D array in size of (num_date, num_row, num_col) temp_coh : 2D array in size of (num_row, num_col) ts_std : 3D array in size of (num_date, num_row, num_col) num_inv_ifgram : 2D array in size of (num_row, num_col) Example: ifgram_inversion_patch('ifgramStack.h5', box=(0,200,1316,400), ref_phase=np.array(), weight_func='fim', mask_dataset_name='coherence') ifgram_inversion_patch('ifgramStack_001.h5', box=None, ref_phase=None, weight_func='fim', mask_dataset_name='coherence') """ stack_obj = ifgramStack(ifgram_file) stack_obj.open(print_msg=False) # Size Info - Patch if box: print('processing %8d/%d lines ...' % (box[3], stack_obj.length)) num_row = box[3] - box[1] num_col = box[2] - box[0] else: num_row = stack_obj.length num_col = stack_obj.width num_pixel = num_row * num_col # Design matrix date12_list = stack_obj.get_date12_list(dropIfgram=True) A, B = stack_obj.get_design_matrix(date12_list=date12_list) num_ifgram = len(date12_list) num_date = A.shape[1] + 1 try: ref_date = str( np.loadtxt('reference_date.txt', dtype=bytes).astype(str)) except: ref_date = stack_obj.dateList[0] ref_idx = stack_obj.dateList.index(ref_date) time_idx = [i for i in range(num_date)] time_idx.remove(ref_idx) Astd = stack_obj.get_design_matrix(refDate=ref_date, dropIfgram=True)[0] # Initialization of output matrix print('number of interferograms: {}'.format(num_ifgram)) print('number of acquisitions : {}'.format(num_date)) print('number of lines : {}'.format(stack_obj.length)) print('number of columns: {}'.format(stack_obj.width)) ts = np.zeros((num_date, num_pixel), np.float32) ts_std = np.zeros((num_date, num_pixel), np.float32) temp_coh = np.zeros(num_pixel, np.float32) num_inv_ifgram = np.zeros(num_pixel, np.int16) # Read/Mask unwrapPhase pha_data = read_unwrap_phase(stack_obj, box, ref_phase, skip_zero_phase=skip_zero_phase) pha_data = mask_unwrap_phase(pha_data, stack_obj, box, mask_ds_name=mask_dataset_name, mask_threshold=mask_threshold) # Mask for pixels to invert mask = np.ones(num_pixel, np.bool_) # 1 - Water Mask if water_mask_file: print(('skip pixels on water with mask from' ' file: {}').format(os.path.basename(water_mask_file))) dsNames = readfile.get_dataset_list(water_mask_file) dsName = [i for i in dsNames if i in ['waterMask', 'mask']][0] waterMask = readfile.read(water_mask_file, datasetName=dsName, box=box)[0].flatten() mask *= np.array(waterMask, np.bool_) del waterMask # 2 - Mask for Zero Phase in ALL ifgrams print('skip pixels with zero/nan value in all interferograms') phase_stack = np.nanmean(pha_data, axis=0) mask *= np.multiply(~np.isnan(phase_stack), phase_stack != 0.) del phase_stack # Invert pixels on mask 1+2 num_pixel2inv = int(np.sum(mask)) idx_pixel2inv = np.where(mask)[0] print(('number of pixels to invert: {} out of {}' ' ({:.1f}%)').format(num_pixel2inv, num_pixel, num_pixel2inv / num_pixel * 100)) if num_pixel2inv < 1: ts = ts.reshape(num_date, num_row, num_col) temp_coh = temp_coh.reshape(num_row, num_col) ts_std = ts_std.reshape(num_date, num_row, num_col) num_inv_ifgram = num_inv_ifgram.reshape(num_row, num_col) return ts, temp_coh, ts_std, num_inv_ifgram # Inversion - SBAS if weight_func == 'sbas': # get tbase_diff (for SBAS approach) date_list = stack_obj.get_date_list(dropIfgram=True) tbase = np.array(ptime.date_list2tbase(date_list)[0], np.float32) / 365.25 tbase_diff = np.diff(tbase).reshape(-1, 1) # Mask for Non-Zero Phase in ALL ifgrams (share one B in sbas inversion) mask_all_net = np.all(pha_data, axis=0) mask_all_net *= mask mask_part_net = mask ^ mask_all_net if np.sum(mask_all_net) > 0: print(('inverting pixels with valid phase in all ifgrams' ' ({:.0f} pixels) ...').format(np.sum(mask_all_net))) # num_all_net = int(np.sum(mask_all_net)) # pha_data_temp = pha_data[:, mask_all_net] # ts1 = np.zeros((num_date-1, num_all_net)) # temp_coh1 = np.zeros(num_all_net) # step = 1000 # loop_num = int(np.floor(num_all_net/step)) # prog_bar = ptime.progressBar(maxValue=loop_num) # for i in range(loop_num): # [i0, i1] = [i * step, min((i + 1) * step, num_all_net)] # ts1[:, i0:i1], temp_coh1[i0:i1] = network_inversion_sbas(B, # ifgram=pha_data_temp[:, i0:i1], # tbase_diff=tbase_diff, # skip_zero_phase=False) # prog_bar.update(i+1, suffix=i0) # prog_bar.close() ts1, temp_coh1, ifg_num1 = network_inversion_sbas( B, ifgram=pha_data[:, mask_all_net], tbase_diff=tbase_diff, skip_zero_phase=False) ts[1:, mask_all_net] = ts1 temp_coh[mask_all_net] = temp_coh1 num_inv_ifgram[mask_all_net] = ifg_num1 if np.sum(mask_part_net) > 0: print(('inverting pixels with valid phase in some ifgrams' ' ({:.0f} pixels) ...').format(np.sum(mask_part_net))) num_pixel2inv = int(np.sum(mask_part_net)) idx_pixel2inv = np.where(mask_part_net)[0] prog_bar = ptime.progressBar(maxValue=num_pixel2inv) for i in range(num_pixel2inv): idx = idx_pixel2inv[i] ts1, temp_coh1, ifg_num1 = network_inversion_sbas( B, ifgram=pha_data[:, idx], tbase_diff=tbase_diff, skip_zero_phase=skip_zero_phase) ts[1:, idx] = ts1.flatten() temp_coh[idx] = temp_coh1 num_inv_ifgram[idx] = ifg_num1 prog_bar.update(i + 1, every=100, suffix='{}/{} pixels'.format( i + 1, num_pixel2inv)) prog_bar.close() # Inversion - WLS else: weight = read_coherence2weight(stack_obj, box=box, weight_func=weight_func) # Converting to 32 bit floats leads to 2X speedup # (comment it out as we now convert it beforehand) # A = np.array(A, np.float32) # pha_data = np.array(pha_data, np.float32) # weight = np.array(weight, np.float32) # Astd = np.array(Astd, np.float32) # Weighted Inversion pixel by pixel print('inverting network of interferograms into time series ...') prog_bar = ptime.progressBar(maxValue=num_pixel2inv) for i in range(num_pixel2inv): idx = idx_pixel2inv[i] ts1, temp_coh1, ts_std1, ifg_numi = network_inversion_wls( A, ifgram=pha_data[:, idx], weight=weight[:, idx], skip_zero_phase=skip_zero_phase, Astd=Astd) ts[1:, idx] = ts1.flatten() temp_coh[idx] = temp_coh1 ts_std[time_idx, idx] = ts_std1.flatten() num_inv_ifgram[idx] = ifg_numi prog_bar.update(i + 1, every=100, suffix='{}/{} pixels'.format(i + 1, num_pixel2inv)) prog_bar.close() ts = ts.reshape(num_date, num_row, num_col) ts_std = ts_std.reshape(num_date, num_row, num_col) temp_coh = temp_coh.reshape(num_row, num_col) num_inv_ifgram = num_inv_ifgram.reshape(num_row, num_col) # write output files if input file is splitted (box == None) if box is None: # metadata metadata = dict(stack_obj.metadata) metadata[key_prefix + 'weightFunc'] = weight_func suffix = re.findall('_\d{3}', ifgram_file)[0] write2hdf5_file(ifgram_file, metadata, ts, temp_coh, ts_std, num_inv_ifgram, suffix) return else: return ts, temp_coh, ts_std, num_inv_ifgram
def correct_dem_error(inps, A_def): """Correct DEM error of input timeseries file""" # Read Date Info ts_obj = timeseries(inps.timeseries_file) ts_obj.open() num_date = ts_obj.numDate num_pixel = ts_obj.numPixel tbase = np.array(ts_obj.tbase, np.float32) / 365.25 num_step = len(inps.stepFuncDate) drop_date, inps.excludeDate = read_exclude_date(inps.excludeDate, ts_obj.dateList) if inps.polyOrder > np.sum(drop_date): raise ValueError(("input poly order {} > number of acquisition {}!" " Reduce it!").format(inps.polyOrder, np.sum(drop_date))) # Read time-series Data print('reading time-series data ...') ts_data = ts_obj.read().reshape((num_date, -1)) ##-------------------------------- Loop for L2-norm inversion --------------------------------## print('inverting DEM error ...') delta_z = np.zeros(num_pixel, dtype=np.float32) ts_cor = np.zeros((num_date, num_pixel), dtype=np.float32) ts_res = np.zeros((num_date, num_pixel), dtype=np.float32) if num_step > 0: step_model = np.zeros((num_step, num_pixel), dtype=np.float32) print('skip pixels with zero/NaN value in all acquisitions') ts_mean = np.nanmean(ts_data, axis=0) mask = np.multiply(~np.isnan(ts_mean), ts_mean != 0.) del ts_mean if inps.rangeDist.size == 1: A_geom = inps.pbase / (inps.rangeDist * inps.sinIncAngle) A = np.hstack((A_geom, A_def)) ts_data = ts_data[:, mask] (delta_z_i, ts_cor_i, ts_res_i, step_model_i) = estimate_dem_error(ts_data, A, tbase=tbase, drop_date=drop_date, phaseVelocity=inps.phaseVelocity, num_step=num_step) delta_z[mask] = delta_z_i ts_cor[:, mask] = ts_cor_i ts_res[:, mask] = ts_res_i if num_step > 0: step_model[:, mask] = step_model_i else: # mask print( 'skip pixels with zero/nan value in geometry: incidence angle or range distance' ) mask *= np.multiply(inps.sinIncAngle != 0., inps.rangeDist != 0.) num_pixel2inv = np.sum(mask) idx_pixel2inv = np.where(mask)[0] print(('number of pixels to invert: {} out of {}' ' ({:.1f}%)').format(num_pixel2inv, num_pixel, num_pixel2inv / num_pixel * 100)) # update data matrix to save memory and IO ts_data = ts_data[:, mask] inps.rangeDist = inps.rangeDist[mask] inps.sinIncAngle = inps.sinIncAngle[mask] if inps.pbase.shape[1] != 1: inps.pbase = inps.pbase[:, mask] # loop pixel by pixel prog_bar = ptime.progressBar(maxValue=num_pixel2inv) for i in range(num_pixel2inv): prog_bar.update(i + 1, every=1000, suffix='{}/{}'.format(i + 1, num_pixel2inv)) idx = idx_pixel2inv[i] # design matrix if inps.pbase.shape[1] == 1: pbase = inps.pbase else: pbase = inps.pbase[:, i].reshape(-1, 1) A_geom = pbase / (inps.rangeDist[i] * inps.sinIncAngle[i]) A = np.hstack((A_geom, A_def)) (delta_z_i, ts_cor_i, ts_res_i, step_model_i) = estimate_dem_error( ts_data[:, i], A, tbase=tbase, drop_date=drop_date, phaseVelocity=inps.phaseVelocity, num_step=num_step) delta_z[idx:idx + 1] = delta_z_i ts_cor[:, idx:idx + 1] = ts_cor_i ts_res[:, idx:idx + 1] = ts_res_i if num_step > 0: step_model[:, idx:idx + 1] = step_model_i prog_bar.close() del ts_data ##---------------------------------------- Output -----------------------------------------## # prepare for output ts_cor = ts_cor.reshape((num_date, ts_obj.length, ts_obj.width)) ts_res = ts_res.reshape((num_date, ts_obj.length, ts_obj.width)) delta_z = delta_z.reshape((ts_obj.length, ts_obj.width)) if num_step > 0: step_model = step_model.reshape( (num_step, ts_obj.length, ts_obj.width)) atr = dict(ts_obj.metadata) # config parameter print( 'add/update the following configuration metadata to file:\n{}'.format( configKeys)) for key in configKeys: atr[key_prefix + key] = str(vars(inps)[key]) # 1. Estimated DEM error outfile = 'demErr.h5' atr['FILE_TYPE'] = 'dem' atr['UNIT'] = 'm' writefile.write(delta_z, out_file=outfile, metadata=atr) # 2. Time-series corrected for DEM error atr['FILE_TYPE'] = 'timeseries' writefile.write(ts_cor, out_file=inps.outfile, metadata=atr, ref_file=ts_obj.file) # 3. Time-series of inversion residual ts_ref_file = os.path.join(os.path.dirname(inps.outfile), 'timeseriesResidual.h5') writefile.write(ts_res, out_file=ts_ref_file, metadata=atr, ref_file=ts_obj.file) # 4. Time-series of estimated Step Model if num_step > 0: atr.pop('REF_DATE') step_obj = timeseries( os.path.join(os.path.dirname(inps.outfile), 'timeseriesStepModel.h5')) step_obj.write2hdf5(data=step_model, metadata=atr, dates=inps.stepFuncDate) ## 5. Time-series of estimated Deformation Model = poly model + step model #ts_def_obj = timeseries(os.path.join(os.path.dirname(inps.outfile), 'timeseriesDefModel.h5')) #ts_def_obj.write2hdf5(data=ts_cor - ts_res, refFile=ts_obj.file) #del ts_cor, ts_res return inps
def unwrap_error_correction_phase_closure(ifgram_file, mask_file, ifgram_cor_file=None): """Correct unwrapping errors in network of interferograms using phase closure. Inputs: ifgram_file - string, name/path of interferograms file mask_file - string, name/path of mask file to mask the pixels to be corrected ifgram_cor_file - string, optional, name/path of corrected interferograms file Output: ifgram_cor_file Example: 'unwrapIfgram_unwCor.h5' = unwrap_error_correction_phase_closure('Seeded_unwrapIfgram.h5','mask.h5') """ print('read mask from file: '+mask_file) mask = readfile.read(mask_file, datasetName='mask')[0].flatten(1) atr = readfile.read_attribute(ifgram_file) length = int(atr['LENGTH']) width = int(atr['WIDTH']) k = atr['FILE_TYPE'] pixel_num = length*width # Check reference pixel try: ref_y = int(atr['REF_Y']) ref_x = int(atr['REF_X']) print('reference pixel in y/x: %d/%d' % (ref_y, ref_x)) except: sys.exit('ERROR: Can not find ref_y/x value, input file is not referenced in space!') h5 = h5py.File(ifgram_file, 'r') ifgram_list = sorted(h5[k].keys()) ifgram_num = len(ifgram_list) # Prepare curls curls, Triangles, C = ut.get_triangles(h5) curl_num = np.shape(curls)[0] print('Number of triangles: ' + str(curl_num)) curl_file = 'curls.h5' if not os.path.isfile(curl_file): print('writing >>> '+curl_file) ut.generate_curls(curl_file, h5, Triangles, curls) thr = 0.50 curls = np.array(curls) n1 = curls[:, 0] n2 = curls[:, 1] n3 = curls[:, 2] print('reading interferograms...') print('Number of interferograms: ' + str(ifgram_num)) data = np.zeros((ifgram_num, pixel_num), np.float32) prog_bar = ptime.progressBar(maxValue=ifgram_num) for ni in range(ifgram_num): ifgram = ifgram_list[ni] d = h5[k][ifgram].get(ifgram)[:].flatten(1) data[ni, :] = d prog_bar.update(ni+1) prog_bar.close() print('reading curls ...') print('number of culrs: '+str(curl_num)) h5curl = h5py.File(curl_file, 'r') curl_list = sorted(h5curl[k].keys()) curl_data = np.zeros((curl_num, pixel_num), np.float32) prog_bar = ptime.progressBar(maxValue=curl_num) for ni in range(curl_num): d = h5curl[k][curl_list[ni]].get(curl_list[ni])[:].flatten(1) curl_data[ni, :] = d.flatten(1) prog_bar.update(ni+1) prog_bar.close() h5curl.close() print('estimating unwrapping error pixel by pixel ...') EstUnwrap = np.zeros((ifgram_num, pixel_num), np.float32) prog_bar = ptime.progressBar(maxValue=pixel_num) for ni in range(pixel_num): if mask[ni] == 1: dU = data[:, ni] unwCurl = np.array(curl_data[:, ni]) ind = np.abs(unwCurl) >= thr N1 = n1[ind] N2 = n2[ind] N3 = n3[ind] indC = np.abs(unwCurl) < thr Nc1 = n1[indC] Nc2 = n2[indC] Nc3 = n3[indC] N = np.hstack([N1, N2, N3]) UniN = np.unique(N) Nc = np.hstack([Nc1, Nc2, Nc3]) UniNc = np.unique(Nc) inter = list(set(UniNc) & set(UniN)) # intersetion UniNc = list(UniNc) for x in inter: UniNc.remove(x) D = np.zeros([len(UniNc), ifgram_num]) for i in range(len(UniNc)): D[i, UniNc[i]] = 1 AAA = np.vstack([-2*np.pi*C, D]) AAAA = np.vstack([AAA, 0.25*np.eye(ifgram_num)]) ########## # with Tikhonov regularization: LLL = list(np.dot(C, dU)) + list(np.zeros(np.shape(UniNc)[0])) + list(np.zeros(ifgram_num)) ind = np.isnan(AAAA) M1 = pinv(AAAA) M = np.dot(M1, LLL) EstUnwrap[:, ni] = np.round(M[0:ifgram_num])*2.0*np.pi prog_bar.update(ni+1, suffix='%s/%d' % (ni, pixel_num)) prog_bar.close() dataCor = data + EstUnwrap # Output if not ifgram_cor_file: ifgram_cor_file = os.path.splitext(ifgram_file)[0]+'_unwCor.h5' print('writing >>> '+ifgram_cor_file) h5unwCor = h5py.File(ifgram_cor_file, 'w') gg = h5unwCor.create_group(k) prog_bar = ptime.progressBar(maxValue=ifgram_num) for i in range(ifgram_num): ifgram = ifgram_list[i] group = gg.create_group(ifgram) dset = group.create_dataset(ifgram, data=np.reshape(dataCor[i, :], [width, length]).T) for key, value in h5[k][ifgram].attrs.items(): group.attrs[key] = value prog_bar.update(i+1) prog_bar.close() h5unwCor.close() h5.close() return ifgram_cor_file
def unwrap_error_correction_bridging(ifgram_file, mask_file, y_list, x_list, ramp_type='plane', ifgram_cor_file=None, save_cor_deramp_file=False): """Unwrapping error correction with bridging. Inputs: ifgram_file : string, name/path of interferogram(s) to be corrected mask_file : string, name/path of mask file to mark different patches y/x_list : list of int, bonding points in y/x ifgram_cor_file : string, optional, output file name save_cor_deramp_file : bool, optional Output: ifgram_cor_file Example: y_list = [235, 270, 350, 390] x_list = [880, 890, 1200, 1270] unwrap_error_correction_bridging('unwrapIfgram.h5', 'mask_all.h5', y_list, x_list, 'quadratic') """ ##### Mask and Ramp mask = readfile.read(mask_file, datasetName='mask')[0] ramp_mask = mask == 1 print('estimate phase ramp during the correction') print('ramp type: '+ramp_type) # Bridge Info # Check for i in range(len(x_list)): if mask[y_list[i], x_list[i]] == 0: print('\nERROR: Connecting point (%d,%d) is out of masked area! Select them again!\n' % (y_list[i], x_list[i])) sys.exit(1) print('Number of bridges: '+str(len(x_list)/2)) print('Bonding points coordinates:\nx: '+str(x_list)+'\ny: '+str(y_list)) # Plot Connecting Pair of Points plot_bonding_points = False if plot_bonding_points: point_yx = '' line_yx = '' n_bridge = len(x)/2 for i in range(n_bridge): pairs_yx = '{},{},{},{}'.format(y[2*i], x[2*i], y[2*i+1], x[2*i+1]) if not i == n_bridge-1: point_yx += pair_yx+',' line_yx += pair_yx+';' else: point_yx += pair_yx line_yx += pair_yx try: plot_cmd = 'view.py --point-yx="'+point_yx+'" --line-yx="'+line_yx +\ '" --nodisplay -o bonding_points.png -f '+maskFile print(plot_cmd) os.system(plot_cmd) except: pass # Basic info ext = os.path.splitext(ifgram_file)[1] atr = readfile.read_attribute(ifgram_file) k = atr['FILE_TYPE'] try: ref_y = int(atr['REF_Y']) ref_x = int(atr['REF_X']) print('reference pixel in y/x: %d/%d' % (ref_y, ref_x)) except: sys.exit('ERROR: Can not find ref_y/x value, input file is not referenced in space!') # output file name if not ifgram_cor_file: ifgram_cor_file = os.path.splitext(ifgram_file)[0]+'_unwCor'+ext ifgram_cor_deramp_file = os.path.splitext(ifgram_cor_file)[0]+'_'+ramp_type+ext # HDF5 file if ext == '.h5': # Read h5 = h5py.File(ifgram_file, 'r') ifgram_list = sorted(h5[k].keys()) ifgram_num = len(ifgram_list) h5out = h5py.File(ifgram_cor_file, 'w') group = h5out.create_group(k) print('writing >>> '+ifgram_cor_file) if save_cor_deramp_file: h5out_deramp = h5py.File(ifgram_cor_deramp_file, 'w') group_deramp = h5out_deramp.create_group(k) print('writing >>> '+ifgram_cor_deramp_file) # Loop print('Number of interferograms: '+str(ifgram_num)) prog_bar = ptime.progressBar(maxValue=ifgram_num) date12_list = ptime.list_ifgram2date12(ifgram_list) for i in range(ifgram_num): ifgram = ifgram_list[i] data = h5[k][ifgram].get(ifgram)[:] data -= data[ref_y, ref_x] data_deramp, ramp = deramp.remove_data_surface(data, ramp_mask, ramp_type) data_derampCor = bridging_data(data_deramp, mask, x_list, y_list) ramp[data == 0.] = 0. gg = group.create_group(ifgram) dset = gg.create_dataset(ifgram, data=data_derampCor+ramp) for key, value in h5[k][ifgram].attrs.items(): gg.attrs[key] = value if save_cor_deramp_file: gg_deramp = group_deramp.create_group(ifgram) dset = gg_deramp.create_dataset(ifgram, data=data_derampCor) for key, value in h5[k][ifgram].attrs.items(): gg_deramp.attrs[key] = value prog_bar.update(i+1, suffix=date12_list[i]) prog_bar.close() h5.close() h5out.close() try: h5out_deramp.close() except: pass # .unw file elif ext == '.unw': print('read '+ifgram_file) data = readfile.read(ifgram_file)[0] data -= data[ref_y, ref_x] data_deramp, ramp = deramp.remove_data_surface(data, ramp_mask, ramp_type) data_derampCor = bridging_data(data_deramp, mask, x_list, y_list) print('writing >>> '+ifgram_cor_file) ramp[data == 0.] = 0. ifgram_cor_file = writefile.write(data_derampCor+ramp, out_file=ifgram_cor_file, metadata=atr) if save_cor_deramp_file: print('writing >>> '+ifgram_cor_deramp_file) ifgram_cor_deramp_file = writefile.write(data_derampCor, out_file=ifgram_cor_deramp_file, metadata=atr) else: sys.exit('Un-supported file type: '+ext) return ifgram_cor_file, ifgram_cor_deramp_file
def main(argv): # Inputs try: ifgram_file = argv[0] timeseries_file = argv[1] except: usage() sys.exit(1) try: outfile = argv[2] except: outfile = 'recon_'+ifgram_file atr = readfile.read_attribute(timeseries_file) length = int(atr['LENGTH']) width = int(atr['WIDTH']) # Read time-series file print('loading timeseries ...') h5ts = h5py.File(timeseries_file, 'r') date_list = sorted(h5ts['timeseries'].keys()) date_num = len(date_list) timeseries = np.zeros((date_num, length*width)) print('number of acquisitions: '+str(date_num)) prog_bar = ptime.progressBar(maxValue=date_num) for i in range(date_num): date = date_list[i] d = h5ts['timeseries'].get(date)[:] timeseries[i, :] = d.flatten(0) prog_bar.update(i+1, suffix=date) prog_bar.close() h5ts.close() del d range2phase = -4*np.pi/float(atr['WAVELENGTH']) timeseries = range2phase*timeseries # Estimate interferograms from timeseries print('estimating interferograms from timeseries using design matrix from input interferograms') A, B = ut.design_matrix(ifgram_file) p = -1*np.ones([A.shape[0], 1]) Ap = np.hstack((p, A)) estData = np.dot(Ap, timeseries) del timeseries # Write interferograms file print('writing >>> '+outfile) h5 = h5py.File(ifgram_file, 'r') ifgram_list = sorted(h5['interferograms'].keys()) ifgram_num = len(ifgram_list) date12_list = ptime.list_ifgram2date12(ifgram_list) h5out = h5py.File(outfile, 'w') group = h5out.create_group('interferograms') print('number of interferograms: '+str(ifgram_num)) prog_bar = ptime.progressBar(maxValue=ifgram_num) for i in range(ifgram_num): ifgram = ifgram_list[i] data = np.reshape(estData[i, :], (length, width)) gg = group.create_group(ifgram) dset = gg.create_dataset(ifgram, data=data) for key, value in h5['interferograms'][ifgram].attrs.items(): gg.attrs[key] = value prog_bar.update(i+1, suffix=date12_list[i]) prog_bar.close() h5.close() h5out.close() print('Done.') return outfile
def run_unwrap_error_phase_closure(ifgram_file, common_regions, water_mask_file=None, ccName='connectComponent', dsNameIn='unwrapPhase', dsNameOut='unwrapPhase_phaseClosure'): print('-' * 50) print('correct unwrapping error in {} with phase closure ...'.format( ifgram_file)) stack_obj = ifgramStack(ifgram_file) stack_obj.open() length, width = stack_obj.length, stack_obj.width ref_y, ref_x = stack_obj.refY, stack_obj.refX date12_list = stack_obj.get_date12_list(dropIfgram=False) num_ifgram = len(date12_list) shape_out = (num_ifgram, length, width) # read water mask if water_mask_file and os.path.isfile(water_mask_file): print('read water mask from file:', water_mask_file) water_mask = readfile.read(water_mask_file)[0] else: water_mask = None # prepare output data writing print('open {} with r+ mode'.format(ifgram_file)) f = h5py.File(ifgram_file, 'r+') print('input dataset:', dsNameIn) print('output dataset:', dsNameOut) if dsNameOut in f.keys(): ds = f[dsNameOut] print('access /{d} of np.float32 in size of {s}'.format(d=dsNameOut, s=shape_out)) else: ds = f.create_dataset(dsNameOut, shape_out, maxshape=(None, None, None), chunks=True, compression=None) print('create /{d} of np.float32 in size of {s}'.format(d=dsNameOut, s=shape_out)) # correct unwrap error ifgram by ifgram prog_bar = ptime.progressBar(maxValue=num_ifgram) for i in range(num_ifgram): date12 = date12_list[i] # read unwrap phase to be updated unw_cor = np.squeeze(f[dsNameIn][i, :, :]).astype(np.float32) unw_cor -= unw_cor[ref_y, ref_x] # update kept interferograms only if stack_obj.dropIfgram[i]: # get local region info from connectComponent cc = np.squeeze(f[ccName][i, :, :]) if water_mask is not None: cc[water_mask == 0] = 0 cc_obj = connectComponent(conncomp=cc, metadata=stack_obj.metadata) cc_obj.label() local_regions = measure.regionprops(cc_obj.labelImg) # matching regions and correct unwrap error idx_common = common_regions[0].date12_list.index(date12) for local_reg in local_regions: local_mask = cc_obj.labelImg == local_reg.label U = 0 for common_reg in common_regions: y = common_reg.sample_coords[:, 0] x = common_reg.sample_coords[:, 1] if all(local_mask[y, x]): U = common_reg.int_ambiguity[idx_common] break unw_cor[local_mask] += 2. * np.pi * U # write to hdf5 file ds[i, :, :] = unw_cor prog_bar.update(i + 1, suffix=date12) prog_bar.close() ds.attrs['MODIFICATION_TIME'] = str(time.time()) f.close() print('close {} file.'.format(ifgram_file)) return ifgram_file
def run_deramp(fname, ramp_type, mask_file=None, out_file=None, datasetName=None): """ Remove ramp from each 2D matrix of input file Parameters: fname : str, data file to be derampped ramp_type : str, name of ramp to be estimated. mask_file : str, file of mask of pixels used for ramp estimation out_file : str, output file name datasetName : str, output dataset name, for ifgramStack file type only Returns: out_file : str, output file name """ print('remove {} ramp from file: {}'.format(ramp_type, fname)) if not out_file: fbase, fext = os.path.splitext(fname) out_file = '{}_ramp{}'.format(fbase, fext) start_time = time.time() atr = readfile.read_attribute(fname) # mask if os.path.isfile(mask_file): mask = readfile.read(mask_file, datasetName='mask')[0] print('read mask file: '+mask_file) else: mask = np.ones((int(atr['LENGTH']), int(atr['WIDTH']))) print('use mask of the whole area') # deramping k = atr['FILE_TYPE'] if k == 'timeseries': print('reading data ...') data = readfile.read(fname)[0] print('estimating phase ramp ...') data = deramp(data, mask, ramp_type=ramp_type, metadata=atr)[0] writefile.write(data, out_file, ref_file=fname) elif k == 'ifgramStack': obj = ifgramStack(fname) obj.open(print_msg=False) if not datasetName: datasetName = 'unwrapPhase' with h5py.File(fname, 'a') as f: ds = f[datasetName] dsNameOut = '{}_ramp'.format(datasetName) if dsNameOut in f.keys(): dsOut = f[dsNameOut] print('access HDF5 dataset /{}'.format(dsNameOut)) else: dsOut = f.create_dataset(dsNameOut, shape=(obj.numIfgram, obj.length, obj.width), dtype=np.float32, chunks=True, compression=None) print('create HDF5 dataset /{}'.format(dsNameOut)) prog_bar = ptime.progressBar(maxValue=obj.numIfgram) for i in range(obj.numIfgram): data = ds[i, :, :] data = deramp(data, mask, ramp_type=ramp_type, metadata=atr)[0] dsOut[i, :, :] = data prog_bar.update(i+1, suffix='{}/{}'.format(i+1, obj.numIfgram)) prog_bar.close() print('finished writing to file: '.format(fname)) # Single Dataset File else: data = readfile.read(fname)[0] data = deramp(data, mask, ramp_type, metadata=atr)[0] print('writing >>> {}'.format(out_file)) writefile.write(data, out_file=out_file, ref_file=fname) m, s = divmod(time.time()-start_time, 60) print('time used: {:02.0f} mins {:02.1f} secs.'.format(m, s)) return out_file
def write2hdf5(self, outputFile='ifgramStack.h5', access_mode='w', box=None, compression=None, extra_metadata=None): '''Save/write an ifgramStackDict object into an HDF5 file with the structure below: / Root level Attributes Dictionary for metadata /date 2D array of string in size of (m, 2 ) in YYYYMMDD format for master and slave date /bperp 1D array of float32 in size of (m, ) in meter. /dropIfgram 1D array of bool in size of (m, ). /unwrapPhase 3D array of float32 in size of (m, l, w) in radian. /coherence 3D array of float32 in size of (m, l, w). /connectComponent 3D array of int16 in size of (m, l, w). (optional) /wrapPhase 3D array of float32 in size of (m, l, w) in radian. (optional) /rangeOffset 3D array of float32 in size of (m, l, w). (optional) /azimuthOffset 3D array of float32 in size of (m, l, w). (optional) Parameters: outputFile : str, Name of the HDF5 file for the InSAR stack access_mode : str, access mode of output File, e.g. w, r+ box : tuple, subset range in (x0, y0, x1, y1) extra_metadata : dict, extra metadata to be added into output file Returns: outputFile ''' self.outputFile = outputFile f = h5py.File(self.outputFile, access_mode) print('create HDF5 file {} with {} mode'.format( self.outputFile, access_mode)) self.pairs = sorted([pair for pair in self.pairsDict.keys()]) self.dsNames = list(self.pairsDict[self.pairs[0]].datasetDict.keys()) self.dsNames = [i for i in ifgramDatasetNames if i in self.dsNames] maxDigit = max([len(i) for i in self.dsNames]) self.get_size(box) self.bperp = np.zeros(self.numIfgram) ############################### # 3D datasets containing unwrapPhase, coherence, connectComponent, wrapPhase, etc. for dsName in self.dsNames: dsShape = (self.numIfgram, self.length, self.width) dsDataType = dataType if dsName in ['connectComponent']: dsDataType = np.bool_ print(('create dataset /{d:<{w}} of {t:<25} in size of {s}' ' with compression = {c}').format(d=dsName, w=maxDigit, t=str(dsDataType), s=dsShape, c=str(compression))) ds = f.create_dataset( dsName, shape=dsShape, maxshape=(None, dsShape[1], dsShape[2]), dtype=dsDataType, chunks=(5, 100, 300), #True compression=compression) prog_bar = ptime.progressBar(maxValue=self.numIfgram) for i in range(self.numIfgram): ifgramObj = self.pairsDict[self.pairs[i]] data = ifgramObj.read(dsName, box=box)[0] ds[i, :, :] = data self.bperp[i] = ifgramObj.get_perp_baseline() prog_bar.update(i + 1, suffix='{}_{}'.format(self.pairs[i][0], self.pairs[i][1])) prog_bar.close() ds.attrs['MODIFICATION_TIME'] = str(time.time()) ############################### # 2D dataset containing master and slave dates of all pairs dsName = 'date' dsDataType = np.string_ dsShape = (self.numIfgram, 2) print('create dataset /{d:<{w}} of {t:<25} in size of {s}'.format( d=dsName, w=maxDigit, t=str(dsDataType), s=dsShape)) data = np.array(self.pairs, dtype=dsDataType) ds = f.create_dataset(dsName, data=data) ############################### # 1D dataset containing perpendicular baseline of all pairs dsName = 'bperp' dsDataType = dataType dsShape = (self.numIfgram, ) print('create dataset /{d:<{w}} of {t:<25} in size of {s}'.format( d=dsName, w=maxDigit, t=str(dsDataType), s=dsShape)) data = np.array(self.bperp, dtype=dsDataType) ds = f.create_dataset(dsName, data=data) ############################### # 1D dataset containing bool value of dropping the interferograms or not dsName = 'dropIfgram' dsDataType = np.bool_ dsShape = (self.numIfgram, ) print('create dataset /{d:<{w}} of {t:<25} in size of {s}'.format( d=dsName, w=maxDigit, t=str(dsDataType), s=dsShape)) data = np.ones(dsShape, dtype=dsDataType) dsDate = f.create_dataset(dsName, data=data) ############################### # Attributes self.get_metadata() if extra_metadata: self.metadata.update(extra_metadata) print('add extra metadata: {}'.format(extra_metadata)) self.metadata = ut.subset_attribute(self.metadata, box) self.metadata['FILE_TYPE'] = self.name for key, value in self.metadata.items(): f.attrs[key] = value f.close() print('Finished writing to {}'.format(self.outputFile)) return self.outputFile
def ifgram_inversion_patch(ifgram_file, box=None, ref_phase=None, unwDatasetName='unwrapPhase', weight_func='var', min_norm_velocity=True, mask_dataset_name=None, mask_threshold=0.4, min_redundancy=1.0, water_mask_file=None, skip_zero_phase=True): """Invert one patch of an ifgram stack into timeseries. Parameters: ifgram_file : str, interferograms stack HDF5 file, e.g. ./INPUTS/ifgramStack.h5 box : tuple of 4 int, indicating (x0, y0, x1, y1) pixel coordinate of area of interest or None, to process the whole file and write output file ref_phase : 1D array in size of (num_ifgram) or None weight_func : str, weight function, choose in ['no', 'fim', 'var', 'coh'] mask_dataset_name : str, dataset name in ifgram_file used to mask unwrapPhase pixelwisely mask_threshold : float, min coherence of pixels if mask_dataset_name='coherence' water_mask_file : str, water mask filename if available, skip inversion on water to speed up the process skip_zero_phase : bool, skip zero value of unwrapped phase or not, default yes, for comparison Returns: ts : 3D array in size of (num_date, num_row, num_col) temp_coh : 2D array in size of (num_row, num_col) ts_std : 3D array in size of (num_date, num_row, num_col) num_inv_ifg : 2D array in size of (num_row, num_col) Example: ifgram_inversion_patch('ifgramStack.h5', box=(0,200,1316,400), ref_phase=np.array(), weight_func='var', min_norm_velocity=True, mask_dataset_name='coherence') ifgram_inversion_patch('ifgramStack_001.h5', box=None, ref_phase=None, weight_func='var', min_norm_velocity=True, mask_dataset_name='coherence') """ stack_obj = ifgramStack(ifgram_file) stack_obj.open(print_msg=False) ## debug #y, x = 258, 454 #box = (x, y, x+1, y+1) # Size Info - Patch if box: #print('processing \t %d-%d / %d lines ...' % (box[1], box[3], stack_obj.length)) num_row = box[3] - box[1] num_col = box[2] - box[0] else: num_row = stack_obj.length num_col = stack_obj.width num_pixel = num_row * num_col # get tbase_diff date_list = stack_obj.get_date_list(dropIfgram=True) num_date = len(date_list) tbase = np.array(ptime.date_list2tbase(date_list)[0], np.float32) / 365.25 tbase_diff = np.diff(tbase).reshape(-1, 1) # Design matrix date12_list = stack_obj.get_date12_list(dropIfgram=True) A, B = stack_obj.get_design_matrix4timeseries_estimation( date12_list=date12_list)[0:2] num_ifgram = len(date12_list) # prep for decor std time-series try: ref_date = str( np.loadtxt('reference_date.txt', dtype=bytes).astype(str)) except: ref_date = date_list[0] Astd = stack_obj.get_design_matrix4timeseries_estimation( refDate=ref_date, dropIfgram=True)[0] #ref_idx = date_list.index(ref_date) #time_idx = [i for i in range(num_date)] #time_idx.remove(ref_idx) # Initialization of output matrix ts = np.zeros((num_date, num_pixel), np.float32) ts_std = np.zeros((num_date, num_pixel), np.float32) temp_coh = np.zeros(num_pixel, np.float32) num_inv_ifg = np.zeros(num_pixel, np.int16) # Read/Mask unwrapPhase pha_data = read_unwrap_phase(stack_obj, box, ref_phase, unwDatasetName=unwDatasetName, dropIfgram=True, skip_zero_phase=skip_zero_phase) pha_data = mask_unwrap_phase(pha_data, stack_obj, box, dropIfgram=True, mask_ds_name=mask_dataset_name, mask_threshold=mask_threshold) # Mask for pixels to invert mask = np.ones(num_pixel, np.bool_) # 1 - Water Mask if water_mask_file: print(('skip pixels on water with mask from' ' file: {}').format(os.path.basename(water_mask_file))) atr_msk = readfile.read_attribute(water_mask_file) if (int(atr_msk['LENGTH']), int( atr_msk['WIDTH'])) != (stack_obj.length, stack_obj.width): raise ValueError( 'Input water mask file has different size from ifgramStack file.' ) del atr_msk dsName = [ i for i in readfile.get_dataset_list(water_mask_file) if i in ['waterMask', 'mask'] ][0] waterMask = readfile.read(water_mask_file, datasetName=dsName, box=box)[0].flatten() mask *= np.array(waterMask, np.bool_) del waterMask # 2 - Mask for Zero Phase in ALL ifgrams print('skip pixels with zero/nan value in all interferograms') phase_stack = np.nanmean(pha_data, axis=0) mask *= np.multiply(~np.isnan(phase_stack), phase_stack != 0.) del phase_stack # Invert pixels on mask 1+2 num_pixel2inv = int(np.sum(mask)) idx_pixel2inv = np.where(mask)[0] print(('number of pixels to invert: {} out of {}' ' ({:.1f}%)').format(num_pixel2inv, num_pixel, num_pixel2inv / num_pixel * 100)) if num_pixel2inv < 1: ts = ts.reshape(num_date, num_row, num_col) ts_std = ts_std.reshape(num_date, num_row, num_col) temp_coh = temp_coh.reshape(num_row, num_col) num_inv_ifg = num_inv_ifg.reshape(num_row, num_col) return ts, temp_coh, ts_std, num_inv_ifg # Inversion - SBAS if weight_func in ['no', 'sbas']: # Mask for Non-Zero Phase in ALL ifgrams (share one B in sbas inversion) mask_all_net = np.all(pha_data, axis=0) mask_all_net *= mask mask_part_net = mask ^ mask_all_net if np.sum(mask_all_net) > 0: print(('inverting pixels with valid phase in all ifgrams' ' ({:.0f} pixels) ...').format(np.sum(mask_all_net))) tsi, tcohi, num_ifgi = estimate_timeseries( A, B, tbase_diff, ifgram=pha_data[:, mask_all_net], weight_sqrt=None, min_norm_velocity=min_norm_velocity, skip_zero_phase=skip_zero_phase, min_redundancy=min_redundancy) ts[:, mask_all_net] = tsi temp_coh[mask_all_net] = tcohi num_inv_ifg[mask_all_net] = num_ifgi if np.sum(mask_part_net) > 0: print(('inverting pixels with valid phase in some ifgrams' ' ({:.0f} pixels) ...').format(np.sum(mask_part_net))) num_pixel2inv = int(np.sum(mask_part_net)) idx_pixel2inv = np.where(mask_part_net)[0] prog_bar = ptime.progressBar(maxValue=num_pixel2inv) for i in range(num_pixel2inv): idx = idx_pixel2inv[i] tsi, tcohi, num_ifgi = estimate_timeseries( A, B, tbase_diff, ifgram=pha_data[:, idx], weight_sqrt=None, min_norm_velocity=min_norm_velocity, skip_zero_phase=skip_zero_phase, min_redundancy=min_redundancy) ts[:, idx] = tsi.flatten() temp_coh[idx] = tcohi num_inv_ifg[idx] = num_ifgi prog_bar.update(i + 1, every=1000, suffix='{}/{} pixels'.format( i + 1, num_pixel2inv)) prog_bar.close() # Inversion - WLS else: L = int(stack_obj.metadata['ALOOKS']) * int( stack_obj.metadata['RLOOKS']) weight = read_coherence(stack_obj, box=box, dropIfgram=True) weight = coherence2weight(weight, weight_func=weight_func, L=L, epsilon=5e-2) weight = np.sqrt(weight) # Weighted Inversion pixel by pixel print('inverting network of interferograms into time-series ...') prog_bar = ptime.progressBar(maxValue=num_pixel2inv) for i in range(num_pixel2inv): idx = idx_pixel2inv[i] tsi, tcohi, num_ifgi = estimate_timeseries( A, B, tbase_diff, ifgram=pha_data[:, idx], weight_sqrt=weight[:, idx], min_norm_velocity=min_norm_velocity, skip_zero_phase=skip_zero_phase, min_redundancy=min_redundancy) ts[:, idx] = tsi.flatten() temp_coh[idx] = tcohi num_inv_ifg[idx] = num_ifgi prog_bar.update(i + 1, every=1000, suffix='{}/{} pixels'.format(i + 1, num_pixel2inv)) prog_bar.close() ts = ts.reshape(num_date, num_row, num_col) ts_std = ts_std.reshape(num_date, num_row, num_col) temp_coh = temp_coh.reshape(num_row, num_col) num_inv_ifg = num_inv_ifg.reshape(num_row, num_col) # write output files if input file is splitted (box == None) if box is None: # metadata metadata = dict(stack_obj.metadata) metadata[key_prefix + 'weightFunc'] = weight_func suffix = re.findall('_\d{3}', ifgram_file)[0] write2hdf5_file(ifgram_file, metadata, ts, temp_coh, ts_std, num_inv_ifg, suffix, inps=inps) return else: return ts, temp_coh, ts_std, num_inv_ifg
def remove_surface(fname, surf_type, mask_file=None, out_file=None, ysub=None): start_time = time.time() atr = readfile.read_attribute(fname) k = atr['FILE_TYPE'] print('remove {} ramp from file: '.format(surf_type, fname)) if not out_file: out_file = '{base}_{ramp}{ext}'.format(base=os.path.splitext(fname)[0], ramp=surf_type, ext=os.path.splitext(fname)[1]) if os.path.isfile(mask_file): mask = readfile.read(mask_file, datasetName='mask')[0] print('read mask file: ' + mask_file) else: mask = np.ones((int(atr['LENGTH']), int(atr['WIDTH']))) print('use mask of the whole area') if k == 'timeseries': obj = timeseries(fname) data = obj.read() numDate = data.shape[0] print('estimating phase ramp ...') prog_bar = ptime.progressBar(maxValue=numDate) for i in range(numDate): if not ysub: data[i, :, :] = remove_data_surface(np.squeeze(data[i, :, :]), mask, surf_type)[0] else: data[i, :, :] = remove_data_multiple_surface( np.squeeze(data), mask, surf_type, ysub) prog_bar.update(i + 1, suffix=obj.dateList[i]) prog_bar.close() objOut = timeseries(out_file) objOut.write2hdf5(data=data, refFile=fname) elif k == 'ifgramStack': obj = ifgramStack(fname) obj.open(print_msg=False) with h5py.File(fname, 'a') as f: ds = f['unwrapPhase'] dsName = 'unwrapPhase_{}'.format(surf_type) if dsName in f.keys(): dsOut = f[dsName] print('access HDF5 dataset /{}'.format(dsName)) else: dsOut = f.create_dataset(dsName, shape=(obj.numIfgram, obj.length, obj.width), dtype=np.float32, chunks=True, compression=None) print('create HDF5 dataset /{}'.format(dsName)) prog_bar = ptime.progressBar(maxValue=obj.numIfgram) for i in range(obj.numIfgram): data = ds[i, :, :] mask_n = np.array(mask, np.bool_) mask_n[data == 0.] = 0 if not ysub: data_n = remove_data_surface(data, mask, surf_type)[0] else: data_n = remove_data_multiple_surface( data, mask, surf_type, ysub) dsOut[i, :, :] = data_n prog_bar.update(i + 1, suffix='{}/{}'.format(i + 1, obj.numIfgram)) prog_bar.close() # Single Dataset File else: data, atr = readfile.read(fname) data_n, ramp = remove_data_surface(data, mask, surf_type) writefile.write(data_n, out_file=out_file, metadata=atr) m, s = divmod(time.time() - start_time, 60) print('\ntime used: {:02.0f} mins {:02.1f} secs'.format(m, s)) return out_file