def add_file(fnames, out_file=None): """Generate sum of all input files Parameters: fnames : list of str, path/name of input files to be added out_file : str, optional, path/name of output file Returns: out_file : str, path/name of output file Example: 'mask_all.h5' = add_file(['mask_1.h5','mask_2.h5','mask_3.h5'], 'mask_all.h5') """ # Default output file name ext = os.path.splitext(fnames[0])[1] if not out_file: out_file = os.path.splitext(fnames[0])[0] for i in range(1, len(fnames)): out_file += '_plus_' + os.path.splitext(os.path.basename( fnames[i]))[0] out_file += ext atr = readfile.read_attribute(fnames[0]) dsNames = readfile.get_dataset_list(fnames[0]) dsDict = {} for dsName in dsNames: print('adding {} ...'.format(dsName)) data = readfile.read(fnames[0], datasetName=dsName)[0] for i in range(1, len(fnames)): d = readfile.read(fnames[i], datasetName=dsName)[0] data = add_matrix(data, d) dsDict[dsName] = data writefile.write(dsDict, out_file=out_file, metadata=atr, ref_file=fnames[0]) return out_file
def main(iargs=None): inps = cmd_line_parse(iargs) if not inps.outfile: inps.outfile = os.path.splitext(inps.file)[0]+'.h5' if inps.data_type: if inps.data_type in ['float', 'float32', 'np.float32']: inps.data_type = np.float32 elif inps.data_type in ['float64', 'np.float64']: inps.data_type = np.float64 elif inps.data_type in ['int', 'int16', 'np.int16']: inps.data_type = np.int16 elif inps.data_type in ['bool', 'np.bool_']: inps.data_type = np.bool_ elif inps.data_type in ['complex', 'np.complex64']: inps.data_type = np.complex64 elif inps.data_type in ['complex128', 'np.complex128']: inps.data_type = np.complex128 else: raise ValueError('un-recognized input data type: {}'.format(inps.data_type)) atr = readfile.read_attribute(inps.file) dsNames = readfile.get_dataset_list(inps.file) dsDict = {} for dsName in dsNames: data = readfile.read(inps.file, datasetName=dsName)[0] if inps.data_type: data = np.array(data, inps.data_type) dsDict[dsName] = data writefile.write(dsDict, out_file=inps.outfile, metadata=atr) return inps.outfile
def multilook_file(infile, lks_y, lks_x, outfile=None): lks_y = int(lks_y) lks_x = int(lks_x) # input file info atr = readfile.read_attribute(infile) k = atr['FILE_TYPE'] print('multilooking {} {} file: {}'.format(atr['PROCESSOR'], k, infile)) print('number of looks in y / azimuth direction: %d' % lks_y) print('number of looks in x / range direction: %d' % lks_x) # output file name if not outfile: if os.getcwd() == os.path.dirname(os.path.abspath(infile)): ext = os.path.splitext(infile)[1] outfile = os.path.splitext(infile)[0] + '_' + str( lks_y) + 'alks_' + str(lks_x) + 'rlks' + ext else: outfile = os.path.basename(infile) #print('writing >>> '+outfile) # read source data and multilooking dsNames = readfile.get_dataset_list(infile) maxDigit = max([len(i) for i in dsNames]) dsDict = dict() for dsName in dsNames: print('multilooking {d:<{w}} from {f} ...'.format( d=dsName, w=maxDigit, f=os.path.basename(infile))) data = readfile.read(infile, datasetName=dsName, print_msg=False)[0] data = multilook_data(data, lks_y, lks_x) dsDict[dsName] = data atr = multilook_attribute(atr, lks_y, lks_x) writefile.write(dsDict, out_file=outfile, metadata=atr, ref_file=infile) return outfile
def file_operation(fname, operator, operand, out_file=None): """Mathmathic operation of file""" # Basic Info atr = readfile.read_attribute(fname) k = atr['FILE_TYPE'] print('input is ' + k + ' file: ' + fname) print('operation: file %s %f' % (operator, operand)) # default output filename if not out_file: if operator in ['+', 'plus', 'add', 'addition']: suffix = 'plus' elif operator in ['-', 'minus', 'substract', 'substraction']: suffix = 'minus' elif operator in ['*', 'times', 'multiply', 'multiplication']: suffix = 'multiply' elif operator in ['/', 'obelus', 'divide', 'division']: suffix = 'divide' elif operator in ['^', 'pow', 'power']: suffix = 'pow' out_file = '{}_{}{}{}'.format( os.path.splitext(fname)[0], suffix, str(operand), os.path.splitext(fname)[1]) atr = readfile.read_attribute(fname) dsNames = readfile.get_dataset_list(fname) dsDict = {} for dsName in dsNames: data = readfile.read(fname, datasetName=dsName)[0] data = data_operation(data, operator, operand) dsDict[dsName] = data writefile.write(dsDict, out_file=out_file, metadata=atr, ref_file=fname) return out_file
def mask_file(fname, mask_file, out_file=None, inps=None): """ Mask input fname with mask_file Inputs: fname/mask_file - string, inps_dict - dictionary including the following options: subset_x/y - list of 2 ints, subset in x/y direction threshold - float, threshold/minValue to generate mask Output: out_file - string """ if not inps: inps = cmd_line_parse() if not out_file: out_file = '{}_masked{}'.format( os.path.splitext(fname)[0], os.path.splitext(fname)[1]) # read mask_file mask = readfile.read(mask_file)[0] mask = update_mask_with_inps(mask, inps) # masking input file dsNames = readfile.get_dataset_list(fname) maxDigit = max([len(i) for i in dsNames]) dsDict = {} for dsName in dsNames: if dsName not in ['coherence']: print('masking {d:<{w}} from {f} ...'.format(d=dsName, w=maxDigit, f=fname)) data = readfile.read(fname, datasetName=dsName, print_msg=False)[0] data = mask_matrix(data, mask, fill_value=inps.fill_value) dsDict[dsName] = data writefile.write(dsDict, out_file=out_file, ref_file=fname)
def run_geocode(inps): """geocode all input files""" start_time = time.time() # Prepare geometry for geocoding res_obj = resample(lookupFile=inps.lookupFile, dataFile=inps.file[0], SNWE=inps.SNWE, laloStep=inps.laloStep, processor=inps.processor) res_obj.open() if not inps.nprocs: inps.nprocs = multiprocessing.cpu_count() # resample input files one by one for infile in inps.file: print('-' * 50+'\nresampling file: {}'.format(infile)) atr = readfile.read_attribute(infile, datasetName=inps.dset) outfile = auto_output_filename(infile, inps) if inps.updateMode and ut.run_or_skip(outfile, in_file=[infile, inps.lookupFile]) == 'skip': print('update mode is ON, skip geocoding.') continue # read source data and resample dsNames = readfile.get_dataset_list(infile, datasetName=inps.dset) maxDigit = max([len(i) for i in dsNames]) dsResDict = dict() for dsName in dsNames: print('reading {d:<{w}} from {f} ...'.format(d=dsName, w=maxDigit, f=os.path.basename(infile))) data = readfile.read(infile, datasetName=dsName, print_msg=False)[0] if atr['FILE_TYPE'] == 'timeseries' and len(data.shape) == 2: data = np.reshape(data, (1, data.shape[0], data.shape[1])) res_data = res_obj.run_resample(src_data=data, interp_method=inps.interpMethod, fill_value=inps.fillValue, nprocs=inps.nprocs, print_msg=True) dsResDict[dsName] = res_data # update metadata if inps.radar2geo: atr = metadata_radar2geo(atr, res_obj) else: atr = metadata_geo2radar(atr, res_obj) #if len(dsNames) == 1 and dsName not in ['timeseries']: # atr['FILE_TYPE'] = dsNames[0] # infile = None writefile.write(dsResDict, out_file=outfile, metadata=atr, ref_file=infile) m, s = divmod(time.time()-start_time, 60) print('time used: {:02.0f} mins {:02.1f} secs.\n'.format(m, s)) return outfile
def run_load_data(self, step_name): """Load InSAR stacks into HDF5 files in ./INPUTS folder. It 1) copy auxiliary files into work directory (for Unvi of Miami only) 2) load all interferograms stack files into PYSAR/INPUTS directory. 3) check loading result 4) add custom metadata (optional, for HDF-EOS5 format only) """ # 1) copy aux files (optional) self._copy_aux_file() # 2) loading data scp_args = '--template {}'.format(self.templateFile) if self.customTemplateFile: scp_args += ' {}'.format(self.customTemplateFile) if self.projectName: scp_args += ' --project {}'.format(self.projectName) # run print("load_data.py", scp_args) pysar.load_data.main(scp_args.split()) os.chdir(self.workDir) # 3) check loading result load_complete, stack_file, geom_file = ut.check_loaded_dataset(self.workDir, print_msg=True)[0:3] # 3.1) output waterMask.h5 water_mask_file = 'waterMask.h5' if 'waterMask' in readfile.get_dataset_list(geom_file): print('generate {} from {} for conveniency'.format(water_mask_file, geom_file)) if ut.run_or_skip(out_file=water_mask_file, in_file=geom_file) == 'run': water_mask, atr = readfile.read(geom_file, datasetName='waterMask') atr['FILE_TYPE'] = 'waterMask' writefile.write(water_mask, out_file=water_mask_file, metadata=atr) # 4) add custom metadata (optional) if self.customTemplateFile: print('updating {}, {} metadata based on custom template file: {}'.format( os.path.basename(stack_file), os.path.basename(geom_file), os.path.basename(self.customTemplateFile))) # use ut.add_attribute() instead of add_attribute.py because of # better control of special metadata, such as SUBSET_X/YMIN ut.add_attribute(stack_file, self.customTemplate) ut.add_attribute(geom_file, self.customTemplate) # 5) if not load_complete, plot and raise exception if not load_complete: # plot result if error occured self.plot_result(print_aux=False, plot=plot) # go back to original directory print('Go back to directory:', self.cwd) os.chdir(self.cwd) # raise error msg = 'step {}: NOT all required dataset found, exit.'.format(step_name) raise RuntimeError(msg) return
def filter_file(fname, filter_type, filter_par=None, fname_out=None): """Filter 2D matrix with selected filter Inputs: fname : string, name/path of file to be filtered filter_type : string, filter type filter_par : string, optional, parameter for low/high pass filter for low/highpass_avg, it's kernel size in int for low/highpass_gaussain, it's sigma in float Output: fname_out : string, optional, output file name/path """ # Info filter_type = filter_type.lower() atr = readfile.read_attribute(fname) k = atr['FILE_TYPE'] msg = 'filtering {} file: {} using {} filter'.format(k, fname, filter_type) if filter_type.endswith('avg'): if not filter_par: filter_par = 5 msg += ' with kernel size of {}'.format(filter_par) elif filter_type.endswith('gaussian'): if not filter_par: filter_par = 3.0 msg += ' with sigma of {:.1f}'.format(filter_par) print(msg) # output filename if not fname_out: fname_out = '{}_{}{}'.format( os.path.splitext(fname)[0], filter_type, os.path.splitext(fname)[1]) # filtering file dsNames = readfile.get_dataset_list(fname) maxDigit = max([len(i) for i in dsNames]) dsDict = dict() for dsName in dsNames: msg = 'filtering {d:<{w}} from {f} '.format(d=dsName, w=maxDigit, f=os.path.basename(fname)) data = readfile.read(fname, datasetName=dsName, print_msg=False)[0] if len(data.shape) == 3: num_loop = data.shape[0] for i in range(num_loop): data[i, :, :] = filter_data(data[i, :, :], filter_type, filter_par) sys.stdout.write('\r{} {}/{} ...'.format(msg, i + 1, num_loop)) sys.stdout.flush() print('') else: data = filter_data(data, filter_type, filter_par) dsDict[dsName] = data writefile.write(dsDict, out_file=fname_out, metadata=atr, ref_file=fname) return fname_out
def plot_data(): atr = readfile.read_attribute(file.get()) file_type = atr['FILE_TYPE'] datasets = readfile.get_dataset_list(file.get(), file_type) item = tree.focus() the_item = tree.item(item) epoch_num = the_item['text'] if epoch_num in datasets: view.main([file.get(), epoch_num])
def main(iargs=None): inps = cmd_line_parse(iargs) if not inps.outfile: inps.outfile = os.path.splitext(inps.file)[0] + '.h5' atr = readfile.read_attribute(inps.file) dsNames = readfile.get_dataset_list(inps.file) dsDict = {} for dsName in dsNames: data = readfile.read(inps.file, datasetName=dsName)[0] dsDict[dsName] = data writefile.write(dsDict, out_file=inps.outfile, metadata=atr) return inps.outfile
def run_resample(inps): """resample all input files""" start_time = time.time() # Prepare geometry for geocoding res_obj = resample(lookupFile=inps.lookupFile, dataFile=inps.file[0], SNWE=inps.SNWE, laloStep=inps.laloStep) res_obj.get_geometry_definition() inps.nprocs = multiprocessing.cpu_count() # resample input files one by one for infile in inps.file: print('-' * 50+'\nresampling file: {}'.format(infile)) outfile = auto_output_filename(infile, inps) if inps.updateMode and not ut.update_file(outfile, [infile, inps.lookupFile]): print('update mode is ON, skip geocoding.') return outfile # read source data and resample dsNames = readfile.get_dataset_list(infile, datasetName=inps.dset) maxDigit = max([len(i) for i in dsNames]) dsResDict = dict() for dsName in dsNames: print('resampling {d:<{w}} from {f} using {n} processor cores ...'.format( d=dsName, w=maxDigit, f=os.path.basename(infile), n=inps.nprocs)) data = readfile.read(infile, datasetName=dsName, print_msg=False)[0] res_data = resample_data(data, inps, res_obj) dsResDict[dsName] = res_data # update metadata atr = readfile.read_attribute(infile, datasetName=inps.dset) if inps.radar2geo: atr = metadata_radar2geo(atr, res_obj) else: atr = metadata_geo2radar(atr, res_obj) if len(dsNames) == 1 and dsName not in ['timeseries']: atr['FILE_TYPE'] = dsNames[0] infile = None writefile.write(dsResDict, out_file=outfile, metadata=atr, ref_file=infile) m, s = divmod(time.time()-start_time, 60) print('\ntime used: {:02.0f} mins {:02.1f} secs\nDone.'.format(m, s)) return outfile
def split_ifgram_file(ifgram_file, chunk_size=100e6): stack_obj = ifgramStack(ifgram_file) stack_obj.open(print_msg=False) metadata = dict(stack_obj.metadata) # get reference phase ref_phase = get_ifgram_reference_phase(ifgram_file) # get list of boxes box_list = split_into_boxes(ifgram_file, chunk_size=chunk_size, print_msg=True) num_box = len(box_list) # read/write each patch file outfile_list = [] for i in range(num_box): box = box_list[i] outfile = '{}_{:03d}{}'.format( os.path.splitext(ifgram_file)[0], i + 1, os.path.splitext(ifgram_file)[1]) # datasets print('-' * 50) print('reading all datasets in {} from file: {} ...'.format( box, ifgram_file)) dsNames = readfile.get_dataset_list(ifgram_file) dsDict = {} dsDict['refPhase'] = ref_phase for dsName in dsNames: data = stack_obj.read(datasetName=dsName, box=box, print_msg=False) dsDict[dsName] = data # metadata metadata['LENGTH'] = box[3] - box[1] metadata['WIDTH'] = box[2] - box[0] writefile.write(dsDict, out_file=outfile, metadata=metadata, ref_file=ifgram_file) outfile_list.append(outfile) return outfile_list
def get_geometry_file(dset, geocoded=False, abspath=True, print_msg=True): """Find geometry file containing input specific dataset""" if dset not in geometryDatasetNames: raise ValueError('unrecognized geometry dataset name: {}'.format(dset)) if geocoded: geom_file = './INPUTS/geometryGeo.h5' else: geom_file = './INPUTS/geometryRadar.h5' if not os.path.isfile(geom_file): print('geometry file {} does not exist.'.format(geom_file)) return None if dset not in readfile.get_dataset_list(geom_file): print('dataset {} not found in file {}'.format(dset, geom_file)) return None if abspath: geom_file = os.path.abspath(geom_file) return geom_file
def subset_file(fname, subset_dict_input, out_file=None): """Subset file with Inputs: fname : str, path/name of file out_file : str, path/name of output file subset_dict : dict, subsut parameter, including the following items: subset_x : list of 2 int, subset in x direction, default=None subset_y : list of 2 int, subset in y direction, default=None subset_lat : list of 2 float, subset in lat direction, default=None subset_lon : list of 2 float, subset in lon direction, default=None fill_value : float, optional. filled value for area outside of data coverage. default=None None/not-existed to subset within data coverage only. tight : bool, tight subset or not, for lookup table file, i.e. geomap*.trans Outputs: out_file : str, path/name of output file; out_file = 'subset_'+fname, if fname is in current directory; out_file = fname, if fname is not in the current directory. """ # Input File Info try: atr = readfile.read_attribute(fname) except: return None width = int(atr['WIDTH']) length = int(atr['LENGTH']) k = atr['FILE_TYPE'] print('subset ' + k + ' file: ' + fname + ' ...') subset_dict = subset_dict_input.copy() # Read Subset Inputs into 4-tuple box in pixel and geo coord pix_box, geo_box = subset_input_dict2box(subset_dict, atr) coord = ut.coordinate(atr) # if fill_value exists and not None, subset data and fill assigned value for area out of its coverage. # otherwise, re-check subset to make sure it's within data coverage and initialize the matrix with np.nan outfill = False if 'fill_value' in subset_dict.keys() and subset_dict['fill_value']: outfill = True else: outfill = False if not outfill: pix_box = coord.check_box_within_data_coverage(pix_box) subset_dict['fill_value'] = np.nan geo_box = coord.box_pixel2geo(pix_box) data_box = (0, 0, width, length) print('data range in y/x: ' + str(data_box)) print('subset range in y/x: ' + str(pix_box)) print('data range in lat/lon: ' + str(coord.box_pixel2geo(data_box))) print('subset range in lat/lon: ' + str(geo_box)) if pix_box == data_box: print('Subset range == data coverage, no need to subset. Skip.') return fname # Calculate Subset/Overlap Index pix_box4data, pix_box4subset = get_box_overlap_index(data_box, pix_box) ########################### Data Read and Write ###################### # Output File Name if not out_file: if os.getcwd() == os.path.dirname(os.path.abspath(fname)): if 'tight' in subset_dict.keys() and subset_dict['tight']: out_file = '{}_tight{}'.format( os.path.splitext(fname)[0], os.path.splitext(fname)[1]) else: out_file = 'subset_' + os.path.basename(fname) else: out_file = os.path.basename(fname) print('writing >>> ' + out_file) # subset datasets one by one dsNames = readfile.get_dataset_list(fname) maxDigit = max([len(i) for i in dsNames]) dsDict = dict() for dsName in dsNames: print('subsetting {d:<{w}} from {f} ...'.format( d=dsName, w=maxDigit, f=os.path.basename(fname))) data = readfile.read(fname, datasetName=dsName, print_msg=False)[0] # subset 2D data if len(data.shape) == 2: data_overlap = data[pix_box4data[1]:pix_box4data[3], pix_box4data[0]:pix_box4data[2]] data = np.ones((pix_box[3] - pix_box[1], pix_box[2] - pix_box[0]), data.dtype) * subset_dict['fill_value'] data[pix_box4subset[1]:pix_box4subset[3], pix_box4subset[0]:pix_box4subset[2]] = data_overlap # subset 3D data elif len(data.shape) == 3: data_overlap = data[:, pix_box4data[1]:pix_box4data[3], pix_box4data[0]:pix_box4data[2]] data = np.ones( (data.shape[0], pix_box[3] - pix_box[1], pix_box[2] - pix_box[0]), data.dtype) * subset_dict['fill_value'] data[:, pix_box4subset[1]:pix_box4subset[3], pix_box4subset[0]:pix_box4subset[2]] = data_overlap dsDict[dsName] = data atr = ut.subset_attribute(atr, pix_box) writefile.write(dsDict, out_file=out_file, metadata=atr, ref_file=fname) return out_file
def ifgram_inversion_patch(ifgram_file, box=None, ref_phase=None, weight_func='fim', mask_dataset_name=None, mask_threshold=0.4, water_mask_file=None, skip_zero_phase=True): """Invert one patch of an ifgram stack into timeseries. Parameters: ifgram_file : str, interferograms stack HDF5 file, e.g. ./INPUTS/ifgramStack.h5 box : tuple of 4 int, indicating (x0, y0, x1, y1) pixel coordinate of area of interest or None, to process the whole file and write output file ref_phase : 1D array in size of (num_ifgram) or None weight_func : str, weight function, choose in ['sbas', 'fim', 'var', 'coh'] mask_dataset_name : str, dataset name in ifgram_file used to mask unwrapPhase pixelwisely mask_threshold : float, min coherence of pixels if mask_dataset_name='coherence' water_mask_file : str, water mask filename if available, skip inversion on water to speed up the process skip_zero_phase : bool, skip zero value of unwrapped phase or not, default yes, for comparison Returns: ts : 3D array in size of (num_date, num_row, num_col) temp_coh : 2D array in size of (num_row, num_col) ts_std : 3D array in size of (num_date, num_row, num_col) num_inv_ifgram : 2D array in size of (num_row, num_col) Example: ifgram_inversion_patch('ifgramStack.h5', box=(0,200,1316,400), ref_phase=np.array(), weight_func='fim', mask_dataset_name='coherence') ifgram_inversion_patch('ifgramStack_001.h5', box=None, ref_phase=None, weight_func='fim', mask_dataset_name='coherence') """ stack_obj = ifgramStack(ifgram_file) stack_obj.open(print_msg=False) # Size Info - Patch if box: print('processing %8d/%d lines ...' % (box[3], stack_obj.length)) num_row = box[3] - box[1] num_col = box[2] - box[0] else: num_row = stack_obj.length num_col = stack_obj.width num_pixel = num_row * num_col # Design matrix date12_list = stack_obj.get_date12_list(dropIfgram=True) A, B = stack_obj.get_design_matrix(date12_list=date12_list) num_ifgram = len(date12_list) num_date = A.shape[1] + 1 try: ref_date = str( np.loadtxt('reference_date.txt', dtype=bytes).astype(str)) except: ref_date = stack_obj.dateList[0] ref_idx = stack_obj.dateList.index(ref_date) time_idx = [i for i in range(num_date)] time_idx.remove(ref_idx) Astd = stack_obj.get_design_matrix(refDate=ref_date, dropIfgram=True)[0] # Initialization of output matrix print('number of interferograms: {}'.format(num_ifgram)) print('number of acquisitions : {}'.format(num_date)) print('number of lines : {}'.format(stack_obj.length)) print('number of columns: {}'.format(stack_obj.width)) ts = np.zeros((num_date, num_pixel), np.float32) ts_std = np.zeros((num_date, num_pixel), np.float32) temp_coh = np.zeros(num_pixel, np.float32) num_inv_ifgram = np.zeros(num_pixel, np.int16) # Read/Mask unwrapPhase pha_data = read_unwrap_phase(stack_obj, box, ref_phase, skip_zero_phase=skip_zero_phase) pha_data = mask_unwrap_phase(pha_data, stack_obj, box, mask_ds_name=mask_dataset_name, mask_threshold=mask_threshold) # Mask for pixels to invert mask = np.ones(num_pixel, np.bool_) # 1 - Water Mask if water_mask_file: print(('skip pixels on water with mask from' ' file: {}').format(os.path.basename(water_mask_file))) dsNames = readfile.get_dataset_list(water_mask_file) dsName = [i for i in dsNames if i in ['waterMask', 'mask']][0] waterMask = readfile.read(water_mask_file, datasetName=dsName, box=box)[0].flatten() mask *= np.array(waterMask, np.bool_) del waterMask # 2 - Mask for Zero Phase in ALL ifgrams print('skip pixels with zero/nan value in all interferograms') phase_stack = np.nanmean(pha_data, axis=0) mask *= np.multiply(~np.isnan(phase_stack), phase_stack != 0.) del phase_stack # Invert pixels on mask 1+2 num_pixel2inv = int(np.sum(mask)) idx_pixel2inv = np.where(mask)[0] print(('number of pixels to invert: {} out of {}' ' ({:.1f}%)').format(num_pixel2inv, num_pixel, num_pixel2inv / num_pixel * 100)) if num_pixel2inv < 1: ts = ts.reshape(num_date, num_row, num_col) temp_coh = temp_coh.reshape(num_row, num_col) ts_std = ts_std.reshape(num_date, num_row, num_col) num_inv_ifgram = num_inv_ifgram.reshape(num_row, num_col) return ts, temp_coh, ts_std, num_inv_ifgram # Inversion - SBAS if weight_func == 'sbas': # get tbase_diff (for SBAS approach) date_list = stack_obj.get_date_list(dropIfgram=True) tbase = np.array(ptime.date_list2tbase(date_list)[0], np.float32) / 365.25 tbase_diff = np.diff(tbase).reshape(-1, 1) # Mask for Non-Zero Phase in ALL ifgrams (share one B in sbas inversion) mask_all_net = np.all(pha_data, axis=0) mask_all_net *= mask mask_part_net = mask ^ mask_all_net if np.sum(mask_all_net) > 0: print(('inverting pixels with valid phase in all ifgrams' ' ({:.0f} pixels) ...').format(np.sum(mask_all_net))) # num_all_net = int(np.sum(mask_all_net)) # pha_data_temp = pha_data[:, mask_all_net] # ts1 = np.zeros((num_date-1, num_all_net)) # temp_coh1 = np.zeros(num_all_net) # step = 1000 # loop_num = int(np.floor(num_all_net/step)) # prog_bar = ptime.progressBar(maxValue=loop_num) # for i in range(loop_num): # [i0, i1] = [i * step, min((i + 1) * step, num_all_net)] # ts1[:, i0:i1], temp_coh1[i0:i1] = network_inversion_sbas(B, # ifgram=pha_data_temp[:, i0:i1], # tbase_diff=tbase_diff, # skip_zero_phase=False) # prog_bar.update(i+1, suffix=i0) # prog_bar.close() ts1, temp_coh1, ifg_num1 = network_inversion_sbas( B, ifgram=pha_data[:, mask_all_net], tbase_diff=tbase_diff, skip_zero_phase=False) ts[1:, mask_all_net] = ts1 temp_coh[mask_all_net] = temp_coh1 num_inv_ifgram[mask_all_net] = ifg_num1 if np.sum(mask_part_net) > 0: print(('inverting pixels with valid phase in some ifgrams' ' ({:.0f} pixels) ...').format(np.sum(mask_part_net))) num_pixel2inv = int(np.sum(mask_part_net)) idx_pixel2inv = np.where(mask_part_net)[0] prog_bar = ptime.progressBar(maxValue=num_pixel2inv) for i in range(num_pixel2inv): idx = idx_pixel2inv[i] ts1, temp_coh1, ifg_num1 = network_inversion_sbas( B, ifgram=pha_data[:, idx], tbase_diff=tbase_diff, skip_zero_phase=skip_zero_phase) ts[1:, idx] = ts1.flatten() temp_coh[idx] = temp_coh1 num_inv_ifgram[idx] = ifg_num1 prog_bar.update(i + 1, every=100, suffix='{}/{} pixels'.format( i + 1, num_pixel2inv)) prog_bar.close() # Inversion - WLS else: weight = read_coherence2weight(stack_obj, box=box, weight_func=weight_func) # Converting to 32 bit floats leads to 2X speedup # (comment it out as we now convert it beforehand) # A = np.array(A, np.float32) # pha_data = np.array(pha_data, np.float32) # weight = np.array(weight, np.float32) # Astd = np.array(Astd, np.float32) # Weighted Inversion pixel by pixel print('inverting network of interferograms into time series ...') prog_bar = ptime.progressBar(maxValue=num_pixel2inv) for i in range(num_pixel2inv): idx = idx_pixel2inv[i] ts1, temp_coh1, ts_std1, ifg_numi = network_inversion_wls( A, ifgram=pha_data[:, idx], weight=weight[:, idx], skip_zero_phase=skip_zero_phase, Astd=Astd) ts[1:, idx] = ts1.flatten() temp_coh[idx] = temp_coh1 ts_std[time_idx, idx] = ts_std1.flatten() num_inv_ifgram[idx] = ifg_numi prog_bar.update(i + 1, every=100, suffix='{}/{} pixels'.format(i + 1, num_pixel2inv)) prog_bar.close() ts = ts.reshape(num_date, num_row, num_col) ts_std = ts_std.reshape(num_date, num_row, num_col) temp_coh = temp_coh.reshape(num_row, num_col) num_inv_ifgram = num_inv_ifgram.reshape(num_row, num_col) # write output files if input file is splitted (box == None) if box is None: # metadata metadata = dict(stack_obj.metadata) metadata[key_prefix + 'weightFunc'] = weight_func suffix = re.findall('_\d{3}', ifgram_file)[0] write2hdf5_file(ifgram_file, metadata, ts, temp_coh, ts_std, num_inv_ifgram, suffix) return else: return ts, temp_coh, ts_std, num_inv_ifgram
def ifgram_inversion_patch(ifgram_file, box=None, ref_phase=None, unwDatasetName='unwrapPhase', weight_func='var', min_norm_velocity=True, mask_dataset_name=None, mask_threshold=0.4, min_redundancy=1.0, water_mask_file=None, skip_zero_phase=True): """Invert one patch of an ifgram stack into timeseries. Parameters: ifgram_file : str, interferograms stack HDF5 file, e.g. ./INPUTS/ifgramStack.h5 box : tuple of 4 int, indicating (x0, y0, x1, y1) pixel coordinate of area of interest or None, to process the whole file and write output file ref_phase : 1D array in size of (num_ifgram) or None weight_func : str, weight function, choose in ['no', 'fim', 'var', 'coh'] mask_dataset_name : str, dataset name in ifgram_file used to mask unwrapPhase pixelwisely mask_threshold : float, min coherence of pixels if mask_dataset_name='coherence' water_mask_file : str, water mask filename if available, skip inversion on water to speed up the process skip_zero_phase : bool, skip zero value of unwrapped phase or not, default yes, for comparison Returns: ts : 3D array in size of (num_date, num_row, num_col) temp_coh : 2D array in size of (num_row, num_col) ts_std : 3D array in size of (num_date, num_row, num_col) num_inv_ifg : 2D array in size of (num_row, num_col) Example: ifgram_inversion_patch('ifgramStack.h5', box=(0,200,1316,400), ref_phase=np.array(), weight_func='var', min_norm_velocity=True, mask_dataset_name='coherence') ifgram_inversion_patch('ifgramStack_001.h5', box=None, ref_phase=None, weight_func='var', min_norm_velocity=True, mask_dataset_name='coherence') """ stack_obj = ifgramStack(ifgram_file) stack_obj.open(print_msg=False) ## debug #y, x = 258, 454 #box = (x, y, x+1, y+1) # Size Info - Patch if box: #print('processing \t %d-%d / %d lines ...' % (box[1], box[3], stack_obj.length)) num_row = box[3] - box[1] num_col = box[2] - box[0] else: num_row = stack_obj.length num_col = stack_obj.width num_pixel = num_row * num_col # get tbase_diff date_list = stack_obj.get_date_list(dropIfgram=True) num_date = len(date_list) tbase = np.array(ptime.date_list2tbase(date_list)[0], np.float32) / 365.25 tbase_diff = np.diff(tbase).reshape(-1, 1) # Design matrix date12_list = stack_obj.get_date12_list(dropIfgram=True) A, B = stack_obj.get_design_matrix4timeseries_estimation( date12_list=date12_list)[0:2] num_ifgram = len(date12_list) # prep for decor std time-series try: ref_date = str( np.loadtxt('reference_date.txt', dtype=bytes).astype(str)) except: ref_date = date_list[0] Astd = stack_obj.get_design_matrix4timeseries_estimation( refDate=ref_date, dropIfgram=True)[0] #ref_idx = date_list.index(ref_date) #time_idx = [i for i in range(num_date)] #time_idx.remove(ref_idx) # Initialization of output matrix ts = np.zeros((num_date, num_pixel), np.float32) ts_std = np.zeros((num_date, num_pixel), np.float32) temp_coh = np.zeros(num_pixel, np.float32) num_inv_ifg = np.zeros(num_pixel, np.int16) # Read/Mask unwrapPhase pha_data = read_unwrap_phase(stack_obj, box, ref_phase, unwDatasetName=unwDatasetName, dropIfgram=True, skip_zero_phase=skip_zero_phase) pha_data = mask_unwrap_phase(pha_data, stack_obj, box, dropIfgram=True, mask_ds_name=mask_dataset_name, mask_threshold=mask_threshold) # Mask for pixels to invert mask = np.ones(num_pixel, np.bool_) # 1 - Water Mask if water_mask_file: print(('skip pixels on water with mask from' ' file: {}').format(os.path.basename(water_mask_file))) atr_msk = readfile.read_attribute(water_mask_file) if (int(atr_msk['LENGTH']), int( atr_msk['WIDTH'])) != (stack_obj.length, stack_obj.width): raise ValueError( 'Input water mask file has different size from ifgramStack file.' ) del atr_msk dsName = [ i for i in readfile.get_dataset_list(water_mask_file) if i in ['waterMask', 'mask'] ][0] waterMask = readfile.read(water_mask_file, datasetName=dsName, box=box)[0].flatten() mask *= np.array(waterMask, np.bool_) del waterMask # 2 - Mask for Zero Phase in ALL ifgrams print('skip pixels with zero/nan value in all interferograms') phase_stack = np.nanmean(pha_data, axis=0) mask *= np.multiply(~np.isnan(phase_stack), phase_stack != 0.) del phase_stack # Invert pixels on mask 1+2 num_pixel2inv = int(np.sum(mask)) idx_pixel2inv = np.where(mask)[0] print(('number of pixels to invert: {} out of {}' ' ({:.1f}%)').format(num_pixel2inv, num_pixel, num_pixel2inv / num_pixel * 100)) if num_pixel2inv < 1: ts = ts.reshape(num_date, num_row, num_col) ts_std = ts_std.reshape(num_date, num_row, num_col) temp_coh = temp_coh.reshape(num_row, num_col) num_inv_ifg = num_inv_ifg.reshape(num_row, num_col) return ts, temp_coh, ts_std, num_inv_ifg # Inversion - SBAS if weight_func in ['no', 'sbas']: # Mask for Non-Zero Phase in ALL ifgrams (share one B in sbas inversion) mask_all_net = np.all(pha_data, axis=0) mask_all_net *= mask mask_part_net = mask ^ mask_all_net if np.sum(mask_all_net) > 0: print(('inverting pixels with valid phase in all ifgrams' ' ({:.0f} pixels) ...').format(np.sum(mask_all_net))) tsi, tcohi, num_ifgi = estimate_timeseries( A, B, tbase_diff, ifgram=pha_data[:, mask_all_net], weight_sqrt=None, min_norm_velocity=min_norm_velocity, skip_zero_phase=skip_zero_phase, min_redundancy=min_redundancy) ts[:, mask_all_net] = tsi temp_coh[mask_all_net] = tcohi num_inv_ifg[mask_all_net] = num_ifgi if np.sum(mask_part_net) > 0: print(('inverting pixels with valid phase in some ifgrams' ' ({:.0f} pixels) ...').format(np.sum(mask_part_net))) num_pixel2inv = int(np.sum(mask_part_net)) idx_pixel2inv = np.where(mask_part_net)[0] prog_bar = ptime.progressBar(maxValue=num_pixel2inv) for i in range(num_pixel2inv): idx = idx_pixel2inv[i] tsi, tcohi, num_ifgi = estimate_timeseries( A, B, tbase_diff, ifgram=pha_data[:, idx], weight_sqrt=None, min_norm_velocity=min_norm_velocity, skip_zero_phase=skip_zero_phase, min_redundancy=min_redundancy) ts[:, idx] = tsi.flatten() temp_coh[idx] = tcohi num_inv_ifg[idx] = num_ifgi prog_bar.update(i + 1, every=1000, suffix='{}/{} pixels'.format( i + 1, num_pixel2inv)) prog_bar.close() # Inversion - WLS else: L = int(stack_obj.metadata['ALOOKS']) * int( stack_obj.metadata['RLOOKS']) weight = read_coherence(stack_obj, box=box, dropIfgram=True) weight = coherence2weight(weight, weight_func=weight_func, L=L, epsilon=5e-2) weight = np.sqrt(weight) # Weighted Inversion pixel by pixel print('inverting network of interferograms into time-series ...') prog_bar = ptime.progressBar(maxValue=num_pixel2inv) for i in range(num_pixel2inv): idx = idx_pixel2inv[i] tsi, tcohi, num_ifgi = estimate_timeseries( A, B, tbase_diff, ifgram=pha_data[:, idx], weight_sqrt=weight[:, idx], min_norm_velocity=min_norm_velocity, skip_zero_phase=skip_zero_phase, min_redundancy=min_redundancy) ts[:, idx] = tsi.flatten() temp_coh[idx] = tcohi num_inv_ifg[idx] = num_ifgi prog_bar.update(i + 1, every=1000, suffix='{}/{} pixels'.format(i + 1, num_pixel2inv)) prog_bar.close() ts = ts.reshape(num_date, num_row, num_col) ts_std = ts_std.reshape(num_date, num_row, num_col) temp_coh = temp_coh.reshape(num_row, num_col) num_inv_ifg = num_inv_ifg.reshape(num_row, num_col) # write output files if input file is splitted (box == None) if box is None: # metadata metadata = dict(stack_obj.metadata) metadata[key_prefix + 'weightFunc'] = weight_func suffix = re.findall('_\d{3}', ifgram_file)[0] write2hdf5_file(ifgram_file, metadata, ts, temp_coh, ts_std, num_inv_ifg, suffix, inps=inps) return else: return ts, temp_coh, ts_std, num_inv_ifg
def run_unwrap_error_patch(ifgram_file, box=None, mask_file=None, ref_phase=None, fast_mode=False, thres=0.1, dsNameIn='unwrapPhase'): """Estimate/Correct unwrapping error in ifgram stack on area defined by box. Parameters: ifgram_file : string, ifgramStack file box : tuple of 4 int, indicating areas to be read and analyzed mask_file : string, file name of mask file for pixels to be analyzed ref_pahse : 1D np.array in size of (num_ifgram,) phase value on reference pixel, because: 1) phase value stored in pysar is not reference yet 2) reference point may be out of box definition fast_mode : bool, apply zero jump constraint on ifgrams without unwrapping error. thres : float, threshold of non-zero phase closure to be identified as unwrapping error. Returns: pha_data : 3D np.array in size of (num_ifgram_all, box[3]-box[2], box[2]-box[0]), unwrapped phase value after error correction """ # Basic info stack_obj = ifgramStack(ifgram_file) stack_obj.open(print_msg=False) num_ifgram = stack_obj.numIfgram # Size Info - Patch if box: num_row = box[3] - box[1] num_col = box[2] - box[0] else: num_row = stack_obj.length num_col = stack_obj.width num_pixel = num_row * num_col C = stack_obj.get_design_matrix4ifgram_triangle(dropIfgram=True) print('number of interferograms: {}'.format(C.shape[1])) print('number of triangles: {}'.format(C.shape[0])) # read unwrapPhase pha_data_all = ifginv.read_unwrap_phase(stack_obj, box, ref_phase, unwDatasetName=dsNameIn, dropIfgram=False) pha_data = np.array(pha_data_all[stack_obj.dropIfgram, :]) # mask of pixels to analyze mask = np.ones((num_pixel), np.bool_) print('number of pixels read: {}'.format(num_pixel)) # mask 1. mask of water or area of interest if mask_file: dsNames = readfile.get_dataset_list(mask_file) dsName = [i for i in dsNames if i in ['waterMask', 'mask']][0] waterMask = readfile.read(mask_file, datasetName=dsName, box=box)[0].flatten() mask *= np.array(waterMask, np.bool_) del waterMask print('number of pixels left after mask: {}'.format(np.sum(mask))) # mask 2. mask of pixels without unwrap error: : zero phase closure on all triangles print('calculating phase closure of all possible triangles ...') pha_closure = np.dot(C, pha_data) pha_closure = np.abs(pha_closure - ut.wrap(pha_closure)) # Eq 4.2 (Fattahi, 2015) num_nonzero_closure = np.sum(pha_closure >= thres, axis=0) mask *= (num_nonzero_closure != 0.) del pha_closure print('number of pixels left after checking phase closure: {}'.format(np.sum(mask))) # mask summary num_pixel2proc = int(np.sum(mask)) if num_pixel2proc > 0: ifgram = pha_data[:, mask] ifgram_cor = np.array(ifgram, np.float32) print('number of pixels to process: {} out of {} ({:.2f}%)'.format(num_pixel2proc, num_pixel, num_pixel2proc/num_pixel*100)) # correcting unwrap error based on phase closure print('correcting unwrapping error ...') if fast_mode: ifgram_cor = correct_unwrap_error(ifgram, C, Dconstraint=False)[0] else: prog_bar = ptime.progressBar(maxValue=num_pixel2proc) for i in range(num_pixel2proc): ifgram_cor[:, i] = correct_unwrap_error(ifgram[:, i], C, Dconstraint=True)[0].flatten() prog_bar.update(i+1, every=10, suffix='{}/{}'.format(i+1, num_pixel2proc)) prog_bar.close() pha_data[:, mask] = ifgram_cor pha_data_all[stack_obj.dropIfgram, :] = pha_data pha_data_all = pha_data_all.reshape(num_ifgram, num_row, num_col) num_nonzero_closure = num_nonzero_closure.reshape(num_row, num_col) return pha_data_all, num_nonzero_closure