def get_giant_ifg_list(fnames): m_date_list = [] s_date_list = [] pbase_list = [] ext = os.path.splitext(fnames[0])[1] if ext == '.h5': obj = ifgramStack(fnames[0]) obj.open() m_date_list = obj.mDates[obj.dropIfgram].tolist() s_date_list = obj.sDates[obj.dropIfgram].tolist() pbase_list = obj.pbaseIfgram[obj.dropIfgram].tolist() else: ifgramNum = len(fnames) print('Number of interferograms: %d' % (ifgramNum)) for fname in fnames: atr = readfile.read_attribute(fname) m_date, s_date = ptime.yymmdd(atr['DATE12'].split('-')) pbase = (float(atr['P_BASELINE_TOP_HDR']) + float(atr['P_BASELINE_BOTTOM_HDR'])) / 2. m_date_list.append(m_date) s_date_list.append(s_date) pbase_list.append(pbase) return m_date_list, s_date_list, pbase_list
def split_into_boxes(ifgram_file, chunk_size=100e6, print_msg=True): """Split into chunks in rows to reduce memory usage Parameters: """ shape = ifgramStack(ifgram_file).get_size() # Get r_step / chunk_num r_step = chunk_size / (shape[0] * shape[2]) # split in lines #if inps.weightFunc != 'sbas': # more memory usage (coherence) for WLS # r_step /= 2.0 r_step = int(ceil_to_1(r_step)) chunk_num = int((shape[1] - 1) / r_step) + 1 if print_msg and chunk_num > 1: print('maximum chunk size: %.1E' % chunk_size) print('split %d lines into %d patches for processing' % (shape[1], chunk_num)) print(' with each patch up to %d lines' % r_step) # Computing the inversion box_list = [] for i in range(chunk_num): r0 = i * r_step r1 = min([shape[1], r0 + r_step]) box = (0, r0, shape[2], r1) box_list.append(box) return box_list
def print_date_list(fname, disp_num=False, drop_ifgram=False, print_msg=False): """Print time/date info of file""" atr = readfile.read_attribute(fname) k = atr['FILE_TYPE'] dateList = None if k in ['timeseries']: dateList = timeseries(fname).get_date_list() elif k == 'HDFEOS': obj = HDFEOS(fname) obj.open(print_msg=False) dateList = obj.dateList elif k == 'giantTimeseries': obj = giantTimeseries(fname) obj.open(print_msg=False) dateList = obj.dateList elif k in ['ifgramStack']: dateList = ifgramStack(fname).get_date12_list(dropIfgram=drop_ifgram) elif k in ['giantIfgramStack']: obj = giantIfgramStack(fname) obj.open(print_msg=False) dateList = obj.date12List else: print('--date option can not be applied to {} file, ignore it.'.format(k)) if print_msg and dateList is not None: for i in range(len(dateList)): if disp_num: print('{}\t{}'.format(dateList[i], i)) else: print(dateList[i]) return dateList
def read_network_info(inps): k = readfile.read_attribute(inps.ifgram_file)['FILE_TYPE'] if k != 'ifgramStack': raise ValueError('input file {} is not ifgramStack: {}'.format( inps.ifgram_file, k)) obj = ifgramStack(inps.ifgram_file) obj.open() inps.date12_list = obj.get_date12_list(dropIfgram=False) date12_kept = obj.get_date12_list(dropIfgram=True) inps.ex_date12_list = sorted( list(set(inps.date12_list) - set(date12_kept))) inps.date_list = obj.get_date_list(dropIfgram=False) print('number of all interferograms: {}'.format(len(inps.date12_list))) print('number of dropped interferograms: {}'.format( len(inps.ex_date12_list))) print('number of kept interferograms: {}'.format( len(inps.date12_list) - len(inps.ex_date12_list))) print('number of acquisitions: {}'.format(len(inps.date_list))) if not inps.yx: inps.yx = (obj.refY, obj.refX) print('plot initial coherence matrix at reference pixel: {}'.format( inps.yx)) return inps
def update_object(outFile, inObj, box, updateMode=True): """Do not write h5 file if: 1) h5 exists and readable, 2) it contains all date12 from ifgramStackDict, or all datasets from geometryDict""" write_flag = True if updateMode and ut.run_or_skip(outFile, check_readable=True) == 'skip': if inObj.name == 'ifgramStack': in_size = inObj.get_size(box=box)[1:] in_date12_list = inObj.get_date12_list() outObj = ifgramStack(outFile) out_size = outObj.get_size()[1:] out_date12_list = outObj.get_date12_list(dropIfgram=False) if out_size == in_size and set(in_date12_list).issubset( set(out_date12_list)): print(( 'All date12 exists in file {} with same size as required,' ' no need to re-load.'.format(outFile))) write_flag = False elif inObj.name == 'geometry': outObj = geometry(outFile) outObj.open(print_msg=False) if (outObj.get_size() == inObj.get_size(box=box) and all(i in outObj.datasetNames for i in inObj.get_dataset_list())): print(( 'All datasets exists in file {} with same size as required,' ' no need to re-load.'.format(outFile))) write_flag = False return write_flag
def get_date12_list(File, dropIfgram=False): """Read Date12 info from input file: Pairs.list or multi-group hdf5 file Inputs: File - string, path/name of input multi-group hdf5 file or text file check_drop_ifgram - bool, check the "DROP_IFGRAM" attribute or not for multi-group hdf5 file Output: date12_list - list of string in YYMMDD-YYMMDD format Example: date12List = get_date12_list('unwrapIfgram.h5') date12List = get_date12_list('unwrapIfgram.h5', check_drop_ifgram=True) date12List = get_date12_list('Pairs.list') """ date12_list = [] ext = os.path.splitext(File)[1].lower() if ext == '.h5': k = readfile.read_attribute(File)['FILE_TYPE'] if k == 'ifgramStack': date12_list = ifgramStack(File).get_date12_list( dropIfgram=dropIfgram) else: return None else: txtContent = np.loadtxt(File, dtype=bytes).astype(str) if len(txtContent.shape) == 1: txtContent = txtContent.reshape(-1, 1) date12_list = [i for i in txtContent[:, 0]] date12_list = sorted(date12_list) return date12_list
def get_nonzero_phase_closure(ifgram_file, out_file=None, thres=0.1, unwDatasetName='unwrapPhase'): """Calculate/Read number of non-zero phase closure Parameters: ifgram_file : string, path of ifgram stack file out_file : string, path of num non-zero phase closure file Returns: num_nonzero_closure : 2D np.array in size of (length, width) """ if not out_file: out_file = 'numNonzeroPhaseClosure_{}.h5'.format(unwDatasetName) if os.path.isfile(out_file) and readfile.read_attribute(out_file): print('1. read number of nonzero phase closure from file: {}'.format( out_file)) num_nonzero_closure = readfile.read(out_file)[0] else: obj = ifgramStack(ifgram_file) obj.open(print_msg=False) length, width = obj.length, obj.width ref_phase = obj.get_reference_phase(unwDatasetName=unwDatasetName, dropIfgram=False) C = obj.get_design_matrix4triplet( obj.get_date12_list(dropIfgram=False)) # calculate phase closure line by line to save memory usage num_nonzero_closure = np.zeros((length, width), np.float32) print( '1. calculating phase closure of all pixels from dataset - {} ...'. format(unwDatasetName)) line_step = 10 num_loop = int(np.ceil(length / line_step)) prog_bar = ptime.progressBar(maxValue=num_loop) for i in range(num_loop): # read phase i0, i1 = i * line_step, min(length, (i + 1) * line_step) box = (0, i0, width, i1) pha_data = ifginv.read_unwrap_phase(obj, box, ref_phase, unwDatasetName=unwDatasetName, dropIfgram=False, print_msg=False) # calculate phase closure pha_closure = np.dot(C, pha_data) pha_closure = np.abs(pha_closure - ut.wrap(pha_closure)) # get number of non-zero phase closure num_nonzero = np.sum(pha_closure >= thres, axis=0) num_nonzero_closure[i0:i1, :] = num_nonzero.reshape(i1 - i0, width) prog_bar.update(i + 1, every=1, suffix='{}/{} lines'.format((i + 1) * line_step, length)) prog_bar.close() atr = dict(obj.metadata) atr['FILE_TYPE'] = 'mask' atr['UNIT'] = 1 writefile.write(num_nonzero_closure, out_file=out_file, metadata=atr) return num_nonzero_closure
def read_network_info(inps): k = readfile.read_attribute(inps.ifgram_file)['FILE_TYPE'] if k != 'ifgramStack': raise ValueError('input file {} is not ifgramStack: {}'.format(inps.ifgram_file, k)) obj = ifgramStack(inps.ifgram_file) obj.open(print_msg=inps.print_msg) inps.date12_list = obj.get_date12_list(dropIfgram=False) date12_kept = obj.get_date12_list(dropIfgram=True) inps.ex_date12_list = sorted(list(set(inps.date12_list) - set(date12_kept))) inps.date_list = obj.get_date_list(dropIfgram=False) vprint('number of all interferograms: {}'.format(len(inps.date12_list))) vprint('number of dropped interferograms: {}'.format(len(inps.ex_date12_list))) vprint('number of kept interferograms: {}'.format(len(inps.date12_list) - len(inps.ex_date12_list))) vprint('number of acquisitions: {}'.format(len(inps.date_list))) if inps.lalo: if not inps.lookup_file: lookup_file = os.path.join(os.path.dirname(inps.ifgram_file), 'geometry*.h5') inps.lookup_file = ut.get_lookup_file(filePattern=lookup_file) coord = ut.coordinate(obj.metadata, lookup_file=inps.lookup_file) inps.yx = coord.geo2radar(inps.lalo[0], inps.lalo[1])[0:2] if not inps.yx: inps.yx = (obj.refY, obj.refX) vprint('plot initial coherence matrix at reference pixel: {}'.format(inps.yx)) return inps
def timeseries2ifgram(ts_file, ifgram_file, out_file='reconUnwrapIfgram.h5'): # read time-series atr = readfile.read_attribute(ts_file) range2phase = -4. * np.pi / float(atr['WAVELENGTH']) print('reading timeseries data from file {} ...'.format(ts_file)) ts_data = readfile.read(ts_file)[0] * range2phase num_date, length, width = ts_data.shape ts_data = ts_data.reshape(num_date, -1) # reconstruct unwrapPhase print('reconstructing the interferograms from timeseries') stack_obj = ifgramStack(ifgram_file) stack_obj.open(print_msg=False) A1 = stack_obj.get_design_matrix4timeseries_estimation(dropIfgram=False)[0] num_ifgram = A1.shape[0] A0 = -1. * np.ones((num_ifgram, 1)) A = np.hstack((A0, A1)) ifgram_est = np.dot(A, ts_data).reshape(num_ifgram, length, width) ifgram_est = np.array(ifgram_est, dtype=ts_data.dtype) del ts_data # write to ifgram file dsDict = {} dsDict['unwrapPhase'] = ifgram_est writefile.write(dsDict, out_file=out_file, ref_file=ifgram_file) return ifgram_file
def calculate_temporal_coherence(ifgram_file, timeseries_file, ifg_num_file=None, chunk_size=100e6): """Calculate temporal coherence based on input timeseries file and interferograms file Parameters: ifgram_file : str, path of interferograms file timeseries_file : str, path of time series file ifg_num_file : str, path of file for number of interferograms used in inversion. chunk_size : float Returns: temp_coh : 2D np.array, temporal coherence in float32 """ # get box list and size info box_list = ifginv.split_into_boxes(ifgram_file, chunk_size=chunk_size) num_box = len(box_list) stack_shape = ifgramStack(ifgram_file).get_size() temp_coh = np.zeros(stack_shape[1:3], np.float32) for i in range(num_box): if num_box > 1: print('\n------- Processing Patch %d out of %d --------------' % (i + 1, num_box)) box = box_list[i] temp_cohi = calculate_temporal_coherence_patch( ifgram_file, timeseries_file, box=box, ifg_num_file=ifg_num_file) temp_coh[box[1]:box[3], box[0]:box[2]] = temp_cohi return temp_coh
def get_ifgram_reference_phase(ifgram_file, skip_reference=False): """Read refPhase""" stack_obj = ifgramStack(ifgram_file) stack_obj.get_size() stack_obj.get_metadata() try: ref_y = int(stack_obj.metadata['REF_Y']) ref_x = int(stack_obj.metadata['REF_X']) ref_phase = np.squeeze( stack_obj.read(datasetName='unwrapPhase', box=(ref_x, ref_y, ref_x + 1, ref_y + 1), dropIfgram=False, print_msg=False)) print('reference pixel in y/x: {}'.format((ref_y, ref_x))) except: if skip_reference: ref_phase = np.zeros((stack_obj.numIfgram, ), np.float32) print( 'skip checking reference pixel info - This is for SIMULATION ONLY.' ) else: msg = 'ERROR: No REF_X/Y found! Can not invert interferograms without reference in space.' msg += '\nrun reference_point.py {} for a quick referencing.'.format( stack_obj.file) raise Exception(msg) return ref_phase
def run_unwrap_error_closure(inps, dsNameIn='unwrapPhase', dsNameOut='unwrapPhase_phaseClosure', fast_mode=False): """Run unwrapping error correction in network of interferograms using phase closure. Parameters: inps : Namespace of input arguments including the following: ifgram_file : string, path of ifgram stack file maskFile : string, path of mask file mark the pixels to be corrected dsNameIn : string, dataset name to read in dsnameOut : string, dataset name to write out fast_mode : bool, enable fast processing mode or not Returns: inps.ifgram_file : string, path of corrected ifgram stack file """ print('-'*50) print('Unwrapping Error Coorrection based on Phase Closure Consistency (Fattahi, 2015) ...') if fast_mode: print('fast mode: ON, the following asuumption is ignored for fast processing') print('\tzero phase jump constraint on ifgrams without unwrap error') else: print(('this step can be very slow' 'you could terminate this and try --fast option for quick correction process.')) print('-'*50) stack_obj = ifgramStack(inps.ifgram_file) stack_obj.open() ref_phase = stack_obj.get_reference_phase(unwDatasetName=dsNameIn, dropIfgram=False) num_nonzero_closure = np.zeros((stack_obj.length, stack_obj.width), dtype=np.int16) # split ifgram_file into blocks to save memory num_tri = stack_obj.get_design_matrix4ifgram_triangle(dropIfgram=True).shape[0] length, width = stack_obj.get_size()[1:3] box_list = ifginv.split_into_boxes(dataset_shape=(num_tri, length, width), chunk_size=200e6) num_box = len(box_list) for i in range(num_box): box = box_list[i] if num_box > 1: print('\n------- Processing Patch {} out of {} --------------'.format(i+1, num_box)) # estimate/correct ifgram unw_cor, num_closure = run_unwrap_error_patch(inps.ifgram_file, box=box, mask_file=inps.maskFile, ref_phase=ref_phase, fast_mode=fast_mode, dsNameIn=dsNameIn) num_nonzero_closure[box[1]:box[3], box[0]:box[2]] = num_closure # write ifgram write_hdf5_file_patch(inps.ifgram_file, data=unw_cor, box=box, dsName=dsNameOut) # write number of nonzero phase closure into file num_file = 'numNonzeroPhaseClosure.h5' atr = dict(stack_obj.metadata) atr['FILE_TYPE'] = 'mask' atr['UNIT'] = '1' writefile.write(num_nonzero_closure, out_file=num_file, metadata=atr) print('writing >>> {}'.format(num_file)) return inps.ifgram_file
def main(iargs=None): inps = cmd_line_parse(iargs) if inps.template_file: inps = read_template2inps(inps.template_file, inps) elif all(not i for i in [ inps.referenceFile, inps.tempBaseMax, inps.perpBaseMax, inps.connNumMax, inps.excludeIfgIndex, inps.excludeDate, inps.coherenceBased, inps.startDate, inps.endDate, inps.reset, inps.manual ]): msg = 'No input option found to remove interferogram, exit.\n' msg += 'To manually modify network, please use --manual option ' raise Exception(msg) if inps.reset: print('--------------------------------------------------') reset_network(inps.file) return inps.file inps.date12_to_drop = get_date12_to_drop(inps) if inps.date12_to_drop is not None: ifgramStack(inps.file).update_drop_ifgram( date12List_to_drop=inps.date12_to_drop) print('--------------------------------------------------') ut.nonzero_mask(inps.file) print('--------------------------------------------------') ut.temporal_average(inps.file, datasetName='coherence', updateMode=True) # Touch spatial average txt file of coherence if it's existed ut.touch( os.path.splitext(os.path.basename(inps.file))[0] + '_coherence_spatialAvg.txt') # Plot result if inps.plot: print('\nplot modified network and save to file.') plotCmd = 'plot_network.py {} --nodisplay'.format(inps.file) if inps.template_file: plotCmd += ' --template {}'.format(inps.template_file) print(plotCmd) os.system(plotCmd) print('Done.') return
def calculate_temporal_coherence_patch(ifgram_file, timeseries_file, box=None, ifg_num_file=None): atr = readfile.read_attribute(timeseries_file) if not box: box = (0, 0, int(atr['WIDTH']), int(atr['LENGTH'])) # Read timeseries data ts_obj = timeseries(timeseries_file) ts_obj.open(print_msg=False) print('reading timeseries data from file: {}'.format(timeseries_file)) ts_data = ts_obj.read(box=box, print_msg=False).reshape(ts_obj.numDate, -1) ts_data = ts_data[1:, :] ts_data *= -4 * np.pi / float(atr['WAVELENGTH']) # Read ifgram data stack_obj = ifgramStack(ifgram_file) stack_obj.open(print_msg=False) A = stack_obj.get_design_matrix4timeseries_estimation(dropIfgram=True)[0] print('reading unwrapPhase data from file: {}'.format(ifgram_file)) ifgram_data = stack_obj.read(datasetName='unwrapPhase', box=box).reshape(A.shape[0], -1) ref_value = stack_obj.get_reference_phase(dropIfgram=True).reshape((-1, 1)) ifgram_data -= np.tile(ref_value, (1, ifgram_data.shape[1])) ifgram_diff = ifgram_data - np.dot(A, ts_data) del ts_data pixel_num = ifgram_data.shape[1] temp_coh = np.zeros((pixel_num), np.float32) # (fast) nasty solution, which used all phase value including invalid zero phase if not ifg_num_file: temp_coh = np.abs(np.sum(np.exp(1j * ifgram_diff), axis=0)) / ifgram_diff.shape[0] # (slow) same solution as ifgram_inversion.py, considering: # 1) invalid zero phase in ifgram # 2) design matrix rank deficiency. else: print( 'considering different number of interferograms used in network inversion for each pixel' ) ifg_num_map = readfile.read(ifg_num_file, box=box)[0].flatten() prog_bar = ptime.progressBar(maxValue=pixel_num) for i in range(pixel_num): if ifg_num_map[i] > 0: idx = ifgram_data[:, i] != 0. temp_diff = ifgram_diff[idx, i] temp_coh[i] = np.abs(np.sum(np.exp(1j * temp_diff), axis=0)) / temp_diff.shape[0] prog_bar.update(i + 1, every=1000, suffix='{}/{}'.format(i + 1, pixel_num)) prog_bar.close() temp_coh = np.reshape(temp_coh, (box[3] - box[1], box[2] - box[0])) return temp_coh
def manual_select_pairs_to_remove(stackFile): """Manually select interferograms to remove""" print('\n-------------------------------------------------------------') print('Manually select interferograms to remove') print('1) click two dates/points to select one pair of interferogram') print('2) repeat until you select all pairs you would like to remove') print('3) close the figure to continue the program ...') print('-------------------------------------------------------------\n') obj = ifgramStack(stackFile) obj.open() date12ListAll = obj.date12List pbase = obj.get_perp_baseline_timeseries(dropIfgram=False) dateList = obj.dateList datesNum = mdates.date2num(np.array(ptime.date_list2vector(dateList)[0])) date12ListKept = obj.get_date12_list(dropIfgram=True) date12ListDropped = sorted(list(set(date12ListAll) - set(date12ListKept))) # Display the network fig = plt.figure() ax = fig.add_subplot(111) ax = pp.plot_network(ax, date12ListAll, dateList, list(pbase), date12List_drop=date12ListDropped) print('display the network of interferogram of file: ' + stackFile) date_click = [] date12_click = [] def onclick(event): idx = nearest_neighbor(event.xdata, event.ydata, datesNum, pbase) print('click at ' + dateList[idx]) date_click.append(dateList[idx]) if len(date_click) % 2 == 0 and date_click[-2] != date_click[-1]: [mDate, sDate] = sorted(date_click[-2:]) mIdx = dateList.index(mDate) sIdx = dateList.index(sDate) date12 = mDate + '_' + sDate if date12 in date12ListAll: print('select date12: ' + date12) date12_click.append(date12) ax.plot([datesNum[mIdx], datesNum[sIdx]], [pbase[mIdx], pbase[sIdx]], 'r', lw=4) else: print(date12 + ' is not existed in input file') plt.draw() cid = fig.canvas.mpl_connect('button_press_event', onclick) plt.show() if not ut.yes_or_no('Proceed to drop the ifgrams/date12?'): date12_click = None return date12_click
def write2hdf5_file(ifgram_file, metadata, ts, temp_coh, ts_std=None, num_inv_ifg=None, suffix='', inps=None): stack_obj = ifgramStack(ifgram_file) stack_obj.open(print_msg=False) date_list = stack_obj.get_date_list(dropIfgram=True) # File 1 - timeseries.h5 ts_file = '{}{}.h5'.format(suffix, os.path.splitext(inps.outfile[0])[0]) metadata['REF_DATE'] = date_list[0] metadata['FILE_TYPE'] = 'timeseries' metadata['UNIT'] = 'm' print('-' * 50) print('converting phase to range') phase2range = -1 * float(stack_obj.metadata['WAVELENGTH']) / (4. * np.pi) ts *= phase2range print('calculating perpendicular baseline timeseries') pbase = stack_obj.get_perp_baseline_timeseries(dropIfgram=True) ts_obj = timeseries(ts_file) ts_obj.write2hdf5(data=ts, dates=date_list, bperp=pbase, metadata=metadata) # File 2 - temporalCoherence.h5 out_file = '{}{}.h5'.format(suffix, os.path.splitext(inps.outfile[1])[0]) metadata['FILE_TYPE'] = 'temporalCoherence' metadata['UNIT'] = '1' print('-' * 50) writefile.write(temp_coh, out_file=out_file, metadata=metadata) # File 3 - timeseriesDecorStd.h5 if not np.all(ts_std == 0.): out_file = 'timeseriesDecorStd{}.h5'.format(suffix) metadata['FILE_TYPE'] = 'timeseries' metadata['UNIT'] = 'm' ts_std *= abs(phase2range) print('-' * 50) writefile.write(ts_std, out_file=out_file, metadata=metadata, ref_file=ts_file) # File 4 - numInvIfgram.h5 out_file = 'numInvIfgram{}.h5'.format(suffix) metadata['FILE_TYPE'] = 'mask' metadata['UNIT'] = '1' print('-' * 50) writefile.write(num_inv_ifg, out_file=out_file, metadata=metadata) return
def reset_network(stackFile): """Reset/restore all pairs within the input file by set all DROP_IFGRAM=no""" print("reset dataset 'dropIfgram' to True for all interferograms for file: "+stackFile) obj = ifgramStack(stackFile) obj.open(print_msg=False) if np.all(obj.dropIfgram): print('All dropIfgram are already True, no need to reset.') else: with h5py.File(stackFile, 'r+') as f: f['dropIfgram'][:] = True ut.touch(os.path.splitext(os.path.basename(stackFile))[0]+'_coherence_spatialAvg.txt') return stackFile
def main(iargs=None): inps = cmd_line_parse(iargs) if inps.templateFile: inps = read_template2inps(inps.templateFile, inps) inps.timeseriesFile, inps.tempCohFile = inps.outfile # --fast option if inps.fast: print("Enable fast network inversion.") if inps.weightFunc != 'no': inps.weightFunc = 'no' print("\tforcing weightFunc = 'no'") if inps.maskDataset is not None: inps.maskDataset = None print("\tforcing maskDataset = None") # --dset option if not inps.unwDatasetName: stack_obj = ifgramStack(inps.ifgramStackFile) stack_obj.open(print_msg=False) inps.unwDatasetName = [ i for i in [ 'unwrapPhase_bridging_phaseClosure', 'unwrapPhase_bridging', 'unwrapPhase_phaseClosure', 'unwrapPhase' ] if i in stack_obj.datasetNames ][0] # --update option inps.numIfgram = len( ifgramStack(inps.ifgramStackFile).get_date12_list(dropIfgram=True)) if inps.update_mode and run_or_skip(inps) == 'skip': return inps.outfile # Network Inversion if inps.residualNorm == 'L2': ifgram_inversion(inps.ifgramStackFile, inps) else: raise NotImplementedError('L1 norm minimization is not fully tested.') #ut.timeseries_inversion_L1(inps.ifgramStackFile, inps.timeseriesFile) return inps.outfile
def nonzero_mask(File, out_file='maskConnComp.h5', datasetName=None): """Generate mask file for non-zero value of input multi-group hdf5 file""" atr = readfile.read_attribute(File) k = atr['FILE_TYPE'] if k == 'ifgramStack': mask = ifgramStack(File).nonzero_mask(datasetName=datasetName) else: print('Only ifgramStack file is supported for now, input is '+k) return None atr['FILE_TYPE'] = 'mask' writefile.write(mask, out_file=out_file, metadata=atr) return out_file
def write_hdf5_file_patch(ifgram_file, data, box=None, dsName='unwrapPhase_phaseClosure'): """Write a patch of 3D dataset into an existing h5 file. Parameters: ifgram_file : string, name/path of output hdf5 file data : 3D np.array to be written box : tuple of 4 int, indicating of (x0, y0, x1, y1) of data in file dsName : output dataset name Returns: ifgram_file """ num_ifgram, length, width = ifgramStack(ifgram_file).get_size( dropIfgram=False) if not box: box = (0, 0, width, length) num_row = box[3] - box[1] num_col = box[2] - box[0] # write to existing HDF5 file print('open {} with r+ mode'.format(ifgram_file)) f = h5py.File(ifgram_file, 'r+') # get h5py.Dataset msg = 'dataset /{d} of {t:<10} in size of {s}'.format(d=dsName, t=str(data.dtype), s=(num_ifgram, box[3], box[2])) if dsName in f.keys(): print('update ' + msg) ds = f[dsName] else: print('create ' + msg) ds = f.create_dataset(dsName, (num_ifgram, num_row, num_col), maxshape=(None, None, None), chunks=True, compression=None) # resize h5py.Dataset if current size is not enough if ds.shape != (num_ifgram, length, width): ds.resize((num_ifgram, box[3], box[2])) # write data to file ds[:, box[1]:box[3], box[0]:box[2]] = data ds.attrs['MODIFICATION_TIME'] = str(time.time()) f.close() print('close {}'.format(ifgram_file)) return ifgram_file
def print_date_list(fname, disp_ifgram='all', disp_num=False, print_msg=False): """Print time/date info of file""" k = readfile.read_attribute(fname)['FILE_TYPE'] dateList = None if k in ['timeseries']: dateList = timeseries(fname).get_date_list() elif k == 'HDFEOS': dateList = HDFEOS(fname).get_date_list() elif k == 'giantTimeseries': dateList = giantTimeseries(fname).get_date_list() elif k in ['giantIfgramStack']: dateList = giantIfgramStack(fname).get_date12_list() elif k in ['ifgramStack']: obj = ifgramStack(fname) obj.open(print_msg=False) dateListAll = obj.get_date12_list(dropIfgram=False) dateListKept = obj.get_date12_list(dropIfgram=True) # show dropped ifgram or not if disp_ifgram == 'all': dateList = list(dateListAll) elif disp_ifgram == 'kept': dateList = list(dateListKept) else: dateList = sorted(list(set(dateListAll) - set(dateListKept))) else: print('--date option can not be applied to {} file, ignore it.'.format( k)) # print list info if print_msg and dateList is not None: for d in dateList: if disp_num: if k in ['ifgramStack']: num = dateListAll.index(d) else: num = dateList.index(d) msg = '{}\t{}'.format(d, num) else: msg = d print(msg) return dateList
def read_input_index_list(idxList, stackFile=None): """Read ['2','3:5','10'] into ['2','3','4','5','10']""" idxListOut = [] for idx in idxList: c = sorted([int(i) for i in idx.split(':')]) if len(c) == 2: idxListOut += list(range(c[0], c[1]+1)) elif len(c) == 1: idxListOut.append(c[0]) else: print('Unrecoganized input: '+idx) idxListOut = sorted(set(idxListOut)) if stackFile: obj = ifgramStack(stackFile) obj.open(print_msg=False) idxListOut = [i for i in idxListOut if i < obj.numIfgram] obj.close(print_msg=False) return idxListOut
def main(iargs=None): inps = cmd_line_parse(iargs) if not inps.datasetNameOut: inps.datasetNameOut = '{}_bridging'.format(inps.datasetNameIn) # update mode checking atr = readfile.read_attribute(inps.ifgram_file) if inps.update and atr['FILE_TYPE'] == 'ifgramStack': obj = ifgramStack(inps.ifgram_file) obj.open(print_msg=False) if inps.datasetNameOut in obj.datasetNames: print(("update mode is enabled AND {} already exists" " skip this step.").format(inps.datasetNameOut)) return inps.ifgram_file start_time = time.time() # get mask_cc from phase closure mask_cc_file = detect_unwrap_error(ifgram_file=inps.ifgram_file, mask_file=inps.maskFile, mask_cc_file='maskConnComp.h5', unwDatasetName=inps.datasetNameIn, cutoff=inps.cutoff) # run bridging bridges = search_bridge(mask_cc_file, radius=inps.bridgePtsRadius) run_unwrap_error_bridge(inps.ifgram_file, mask_cc_file, bridges, dsNameIn=inps.datasetNameIn, dsNameOut=inps.datasetNameOut, ramp_type=inps.ramp) if inps.run_closure: print('') inps.datasetNameIn = inps.datasetNameOut inps.datasetNameOut = '{}_phaseClosure'.format(inps.datasetNameIn) run_unwrap_error_closure(inps, dsNameIn=inps.datasetNameIn, dsNameOut=inps.datasetNameOut, fast_mode=True) m, s = divmod(time.time() - start_time, 60) print('\ntime used: {:02.0f} mins {:02.1f} secs\nDone.'.format(m, s)) return inps.ifgram_file
def split_ifgram_file(ifgram_file, chunk_size=100e6): stack_obj = ifgramStack(ifgram_file) stack_obj.open(print_msg=False) metadata = dict(stack_obj.metadata) # get reference phase ref_phase = get_ifgram_reference_phase(ifgram_file) # get list of boxes box_list = split_into_boxes(ifgram_file, chunk_size=chunk_size, print_msg=True) num_box = len(box_list) # read/write each patch file outfile_list = [] for i in range(num_box): box = box_list[i] outfile = '{}_{:03d}{}'.format( os.path.splitext(ifgram_file)[0], i + 1, os.path.splitext(ifgram_file)[1]) # datasets print('-' * 50) print('reading all datasets in {} from file: {} ...'.format( box, ifgram_file)) dsNames = readfile.get_dataset_list(ifgram_file) dsDict = {} dsDict['refPhase'] = ref_phase for dsName in dsNames: data = stack_obj.read(datasetName=dsName, box=box, print_msg=False) dsDict[dsName] = data # metadata metadata['LENGTH'] = box[3] - box[1] metadata['WIDTH'] = box[2] - box[0] writefile.write(dsDict, out_file=outfile, metadata=metadata, ref_file=ifgram_file) outfile_list.append(outfile) return outfile_list
def select_network_candidate(inps): date_list, tbase_list, pbase_list, dop_list = read_baseline_info( baseline_file=inps.baseline_file, reference_file=inps.referenceFile) # Pair selection from reference if inps.referenceFile: log('select initial network from reference file: {}'.format( inps.referenceFile)) stack_obj = ifgramStack(inps.referenceFile) date12_list = stack_obj.get_date12_list(dropIfgram=True) date12_list = ptime.yymmdd_date12(date12_list) # Pais selection from method elif inps.baseline_file: log('select initial network with method: {}'.format(inps.method)) if inps.method == 'all': date12_list = pnet.select_pairs_all(date_list) elif inps.method == 'delaunay': date12_list = pnet.select_pairs_delaunay(date_list, pbase_list, inps.norm) elif inps.method == 'star': date12_list = pnet.select_pairs_star(date_list) elif inps.method == 'sequential': date12_list = pnet.select_pairs_sequential(date_list, inps.connNum) elif inps.method == 'hierarchical': date12_list = pnet.select_pairs_hierarchical( date_list, pbase_list, inps.tempPerpList) elif inps.method == 'mst': date12_list = pnet.select_pairs_mst(date_list, pbase_list) else: raise Exception('Unrecoganized select method: ' + inps.method) log('initial number of interferograms: {}'.format(len(date12_list))) inps.date12_list = date12_list inps.date_list = date_list inps.tbase_list = tbase_list inps.pbase_list = pbase_list inps.dop_list = dop_list return inps
def check_design_matrix(ifgram_file, weight_func='fim'): """Check Rank of Design matrix for weighted inversion""" A = ifgramStack(ifgram_file).get_design_matrix(dropIfgram=True)[0] print( '-------------------------------------------------------------------------------' ) if weight_func == 'sbas': print('generic least square inversion with min-norm phase velocity') print(' based on Berardino et al. (2002, IEEE-TGRS)') print(' OLS for pixels with full rank network') print(' SVD for pixels with rank deficient network') if np.linalg.matrix_rank(A) < A.shape[1]: print( 'WARNING: singular design matrix! Inversion result can be biased!' ) print('continue using its SVD solution on all pixels') else: print( 'weighted least square (WLS) inversion with min-norm phase, pixelwise' ) if np.linalg.matrix_rank(A) < A.shape[1]: print('ERROR: singular design matrix!') print( ' Input network of interferograms is not fully connected!') print(' Can not invert the weighted least square solution.') print('You could try:') print( ' 1) Add more interferograms to make the network fully connected:' ) print(' a.k.a., no multiple subsets nor network islands') print(" 2) Use '-w no' option for non-weighted SVD solution.") raise Exception() print( '-------------------------------------------------------------------------------' ) return A
def check_design_matrix(ifgram_file, weight_func='var'): """Check Rank of Design matrix for weighted inversion""" A = ifgramStack(ifgram_file).get_design_matrix4timeseries_estimation( dropIfgram=True)[0] if weight_func == 'no': if np.linalg.matrix_rank(A) < A.shape[1]: print( 'WARNING: singular design matrix! Inversion result can be biased!' ) print('continue using its SVD solution on all pixels') else: if np.linalg.matrix_rank(A) < A.shape[1]: print('ERROR: singular design matrix!') print( ' Input network of interferograms is not fully connected!') print(' Can not invert the weighted least square solution.') print('You could try:') print( ' 1) Add more interferograms to make the network fully connected:' ) print(' a.k.a., no multiple subsets nor network islands') print(" 2) Use '-w no' option for non-weighted SVD solution.") raise Exception() return A
def get_date_list(fname, print_msg=False): atr = readfile.read_attribute(fname) k = atr['FILE_TYPE'] dateList = None if k in ['timeseries']: obj = timeseries(fname) obj.open(print_msg=False) dateList = obj.dateList elif k in ['ifgramStack']: obj = ifgramStack(fname) obj.open(print_msg=False) dateList = obj.date12List else: print('--date option can not be applied to {} file, ignore it.'.format( k)) try: obj.close(print_msg=False) except: pass if print_msg and dateList is not None: for i in dateList: print(i) return dateList
def check_loaded_dataset(work_dir='./', print_msg=True): """Check the result of loading data for the following two rules: 1. file existance 2. file attribute readability Parameters: work_dir : string, PySAR working directory print_msg : bool, print out message Returns: True, if all required files and dataset exist; otherwise, ERROR If True, PROCESS, SLC folder could be removed. stack_file : geom_file : lookup_file : Example: work_dir = os.path.expandvars('$SCRATCHDIR/SinabungT495F50AlosA/PYSAR') ut.check_loaded_dataset(work_dir) """ load_complete = True if not work_dir: work_dir = os.getcwd() work_dir = os.path.abspath(work_dir) # 1. interferograms stack file: unwrapPhase, coherence flist = [os.path.join(work_dir, 'INPUTS/ifgramStack.h5')] stack_file = is_file_exist(flist, abspath=True) if stack_file is not None: obj = ifgramStack(stack_file) obj.open(print_msg=False) for dname in ['unwrapPhase', 'coherence']: if dname not in obj.datasetNames: raise ValueError( 'required dataset "{}" is missing in file {}'.format( dname, stack_file)) else: raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), './INPUTS/ifgramStack.h5') atr = readfile.read_attribute(stack_file) # 2. geom_file: height if 'X_FIRST' in atr.keys(): flist = [os.path.join(work_dir, 'INPUTS/geometryGeo.h5')] else: flist = [os.path.join(work_dir, 'INPUTS/geometryRadar.h5')] geom_file = is_file_exist(flist, abspath=True) if geom_file is not None: obj = geometry(geom_file) obj.open(print_msg=False) dname = geometryDatasetNames[0] if dname not in obj.datasetNames: raise ValueError( 'required dataset "{}" is missing in file {}'.format( dname, geom_file)) else: raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), './INPUTS/geometry*.h5') # 3. lookup_file: latitude,longitude or rangeCoord,azimuthCoord # could be different than geometry file in case of roipac and gamma flist = [os.path.join(work_dir, 'INPUTS/geometry*.h5')] lookup_file = get_lookup_file(flist, abspath=True, print_msg=print_msg) if lookup_file is not None: obj = geometry(lookup_file) obj.open(print_msg=False) if atr['PROCESSOR'] in ['isce', 'doris']: dnames = [geometryDatasetNames[1], geometryDatasetNames[2]] elif atr['PROCESSOR'] in ['gamma', 'roipac']: dnames = [geometryDatasetNames[3], geometryDatasetNames[4]] else: raise AttributeError('InSAR processor: {}'.format( atr['PROCESSOR'])) for dname in dnames: if dname not in obj.datasetNames: load_complete = False raise Exception( 'required dataset "{}" is missing in file {}'.format( dname, lookup_file)) else: raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), './INPUTS/geometry*.h5') # print message if print_msg: print(('Loaded dataset are processed by ' 'InSAR software: {}'.format(atr['PROCESSOR']))) if 'X_FIRST' in atr.keys(): print('Loaded dataset is in GEO coordinates') else: print('Loaded dataset is in RADAR coordinates') print('Interferograms Stack: {}'.format(stack_file)) print('Geometry File : {}'.format(geom_file)) print('Lookup Table File : {}'.format(lookup_file)) if load_complete: print('-' * 50) print( 'All data needed found/loaded/copied. Processed 2-pass InSAR data can be removed.' ) print('-' * 50) return load_complete, stack_file, geom_file, lookup_file
def get_slice_list(fname): """Get list of 2D slice existed in file (for display)""" fbase, fext = os.path.splitext(os.path.basename(fname)) fext = fext.lower() atr = read_attribute(fname) k = atr['FILE_TYPE'] global slice_list # HDF5 Files if fext in ['.h5', '.he5']: with h5py.File(fname, 'r') as f: d1_list = [i for i in f.keys() if isinstance(f[i], h5py.Dataset)] if k == 'timeseries' and k in d1_list: obj = timeseries(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['geometry'] and k not in d1_list: obj = geometry(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['ifgramStack']: obj = ifgramStack(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['HDFEOS']: obj = HDFEOS(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['giantTimeseries']: obj = giantTimeseries(fname) obj.open(print_msg=False) slice_list = obj.sliceList elif k in ['giantIfgramStack']: obj = giantIfgramStack(fname) obj.open(print_msg=False) slice_list = obj.sliceList else: ## Find slice by walking through the file structure length, width = int(atr['LENGTH']), int(atr['WIDTH']) def get_hdf5_2d_dataset(name, obj): global slice_list if isinstance(obj, h5py.Dataset) and obj.shape[-2:] == (length, width): if obj.ndim == 2: slice_list.append(name) else: warnings.warn( 'file has un-defined {}D dataset: {}'.format( obj.ndim, name)) slice_list = [] with h5py.File(fname, 'r') as f: f.visititems(get_hdf5_2d_dataset) # Binary Files else: if fext.lower() in ['.trans', '.utm_to_rdc']: slice_list = ['rangeCoord', 'azimuthCoord'] elif fbase.startswith('los'): slice_list = ['incidenceAngle', 'azimuthAngle'] elif atr.get('number_bands', '1') == '2' and 'unw' not in k: slice_list = ['band1', 'band2'] else: slice_list = [''] return slice_list