def ifgram_inversion(inps=None): """Phase triangulatino of small baseline interferograms Parameters: inps - namespace Example: inps = cmd_line_parse() ifgram_inversion(inps) """ if not inps: inps = cmd_line_parse() start_time = time.time() ## 1. input info stack_obj = ifgramStack(inps.ifgramStackFile) stack_obj.open(print_msg=False) date12_list = stack_obj.get_date12_list(dropIfgram=True) date_list = stack_obj.get_date_list(dropIfgram=True) length, width = stack_obj.length, stack_obj.width # 1.1 read values on the reference pixel inps.refPhase = stack_obj.get_reference_phase( unwDatasetName=inps.obsDatasetName, skip_reference=inps.skip_ref, dropIfgram=True) # 1.2 design matrix A = stack_obj.get_design_matrix4timeseries(date12_list)[0] num_ifgram, num_date = A.shape[0], A.shape[1] + 1 inps.numIfgram = num_ifgram # 1.3 print key setup info msg = '-------------------------------------------------------------------------------\n' if inps.minNormVelocity: suffix = 'deformation velocity' else: suffix = 'deformation phase' msg += 'least-squares solution with L2 min-norm on: {}\n'.format(suffix) msg += 'minimum redundancy: {}\n'.format(inps.minRedundancy) msg += 'weight function: {}\n'.format(inps.weightFunc) if inps.maskDataset: if inps.maskDataset in ['coherence', 'offsetSNR']: suffix = '{} < {}'.format(inps.maskDataset, inps.maskThreshold) else: suffix = '{} == 0'.format(inps.maskDataset) msg += 'mask out pixels with: {}\n'.format(suffix) else: msg += 'mask: no\n' if np.linalg.matrix_rank(A) < A.shape[1]: msg += '***WARNING: the network is NOT fully connected.\n' msg += '\tInversion result can be biased!\n' msg += '\tContinue to use SVD to resolve the offset between different subsets.\n' msg += '-------------------------------------------------------------------------------' print(msg) print('number of interferograms: {}'.format(num_ifgram)) print('number of acquisitions : {}'.format(num_date)) print('number of lines : {}'.format(length)) print('number of columns : {}'.format(width)) ## 2. prepare output # 2.1 metadata meta = dict(stack_obj.metadata) for key in configKeys: meta[key_prefix + key] = str(vars(inps)[key]) # 2.2 instantiate time-series dsNameDict = { "date": (np.dtype('S8'), (num_date, )), "bperp": (np.float32, (num_date, )), "timeseries": (np.float32, (num_date, length, width)), } meta['FILE_TYPE'] = 'timeseries' meta['UNIT'] = 'm' meta['REF_DATE'] = date_list[0] ts_obj = timeseries(inps.tsFile) ts_obj.layout_hdf5(dsNameDict, meta) # write date time-series date_list_utf8 = [dt.encode('utf-8') for dt in date_list] writefile.write_hdf5_block(inps.tsFile, date_list_utf8, datasetName='date') # write bperp time-series pbase = stack_obj.get_perp_baseline_timeseries(dropIfgram=True) writefile.write_hdf5_block(inps.tsFile, pbase, datasetName='bperp') # 2.3 instantiate temporal coherence dsNameDict = {"temporalCoherence": (np.float32, (length, width))} meta['FILE_TYPE'] = 'temporalCoherence' meta['UNIT'] = '1' meta.pop('REF_DATE') writefile.layout_hdf5(inps.tempCohFile, dsNameDict, metadata=meta) # 2.4 instantiate number of inverted observations dsNameDict = {"mask": (np.float32, (length, width))} meta['FILE_TYPE'] = 'mask' meta['UNIT'] = '1' writefile.layout_hdf5(inps.numInvFile, dsNameDict, metadata=meta) ## 3. run the inversion / estimation and write to disk # 3.1 split ifgram_file into blocks to save memory box_list, num_box = split2boxes(inps.ifgramStackFile, memory_size=inps.memorySize) # 3.2 prepare the input arguments for *_patch() data_kwargs = { "ifgram_file": inps.ifgramStackFile, "ref_phase": inps.refPhase, "obs_ds_name": inps.obsDatasetName, "weight_func": inps.weightFunc, "min_norm_velocity": inps.minNormVelocity, "water_mask_file": inps.waterMaskFile, "mask_ds_name": inps.maskDataset, "mask_threshold": inps.maskThreshold, "min_redundancy": inps.minRedundancy } # 3.3 invert / write block-by-block for i, box in enumerate(box_list): box_width = box[2] - box[0] box_length = box[3] - box[1] if num_box > 1: print('\n------- processing patch {} out of {} --------------'. format(i + 1, num_box)) print('box width: {}'.format(box_width)) print('box length: {}'.format(box_length)) # update box argument in the input data data_kwargs['box'] = box if inps.cluster == 'no': # non-parallel ts, temp_coh, num_inv_ifg = ifgram_inversion_patch( **data_kwargs)[:-1] else: # parallel print('\n\n------- start parallel processing using Dask -------') # initiate the output data ts = np.zeros((num_date, box_length, box_width), np.float32) temp_coh = np.zeros((box_length, box_width), np.float32) num_inv_ifg = np.zeros((box_length, box_width), np.float32) # initiate dask cluster and client cluster_obj = cluster.DaskCluster(inps.cluster, inps.numWorker, config_name=inps.config) cluster_obj.open() # run dask ts, temp_coh, num_inv_ifg = cluster_obj.run( func=ifgram_inversion_patch, func_data=data_kwargs, results=[ts, temp_coh, num_inv_ifg]) # close dask cluster and client cluster_obj.close() print('------- finished parallel processing -------\n\n') # write the block to disk # with 3D block in [z0, z1, y0, y1, x0, x1] # and 2D block in [y0, y1, x0, x1] # time-series - 3D block = [0, num_date, box[1], box[3], box[0], box[2]] writefile.write_hdf5_block(inps.tsFile, data=ts, datasetName='timeseries', block=block) # temporal coherence - 2D block = [box[1], box[3], box[0], box[2]] writefile.write_hdf5_block(inps.tempCohFile, data=temp_coh, datasetName='temporalCoherence', block=block) # number of inverted obs - 2D writefile.write_hdf5_block(inps.numInvFile, data=num_inv_ifg, datasetName='mask', block=block) m, s = divmod(time.time() - start_time, 60) print('time used: {:02.0f} mins {:02.1f} secs.\n'.format(m, s)) # 3.4 update output data on the reference pixel if not inps.skip_ref: # grab ref_y/x ref_y = int(stack_obj.metadata['REF_Y']) ref_x = int(stack_obj.metadata['REF_X']) print('-' * 50) print('update values on the reference pixel: ({}, {})'.format( ref_y, ref_x)) print('set temporal coherence on the reference pixel to 1.') with h5py.File(inps.tempCohFile, 'r+') as f: f['temporalCoherence'][ref_y, ref_x] = 1. print('set # of observations on the reference pixel as {}'.format( num_ifgram)) with h5py.File(inps.numInvFile, 'r+') as f: f['mask'][ref_y, ref_x] = num_ifgram m, s = divmod(time.time() - start_time, 60) print('time used: {:02.0f} mins {:02.1f} secs.\n'.format(m, s)) return
def correct_dem_error(inps): """Correct DEM error of input timeseries file""" start_time = time.time() # limit the number of threads to 1 # for slight speedup and big CPU usage save num_threads_dict = cluster.set_num_threads("1") ## 1. input info # 1.1 read date info ts_obj = timeseries(inps.timeseries_file) ts_obj.open() num_date = ts_obj.numDate length, width = ts_obj.length, ts_obj.width num_step = len(inps.stepFuncDate) # exclude dates date_flag = read_exclude_date(inps.excludeDate, ts_obj.dateList)[0] if inps.polyOrder > np.sum(date_flag): raise ValueError( "input poly order {} > number of acquisition {}! Reduce it!". format(inps.polyOrder, np.sum(date_flag))) # 1.2 design matrix part 1 - time func for surface deformation G_defo = get_design_matrix4defo(inps) ## 2. prepare output # 2.1 metadata meta = dict(ts_obj.metadata) print( 'add/update the following configuration metadata to file:\n{}'.format( configKeys)) for key in configKeys: meta[key_prefix + key] = str(vars(inps)[key]) # 2.2 instantiate est. DEM error dem_err_file = 'demErr.h5' meta['FILE_TYPE'] = 'dem' meta['UNIT'] = 'm' ds_name_dict = {'dem': [np.float32, (length, width), None]} writefile.layout_hdf5(dem_err_file, ds_name_dict, metadata=meta) # 2.3 instantiate corrected time-series ts_cor_file = inps.outfile meta['FILE_TYPE'] = 'timeseries' writefile.layout_hdf5(ts_cor_file, metadata=meta, ref_file=inps.timeseries_file) # 2.4 instantiate residual phase time-series ts_res_file = os.path.join(os.path.dirname(inps.outfile), 'timeseriesResidual.h5') writefile.layout_hdf5(ts_res_file, metadata=meta, ref_file=inps.timeseries_file) ## 3. run the estimation and write to disk # 3.1 split ts_file into blocks to save memory # 1st dimension size: ts (obs / cor / res / step) + dem_err/inc_angle/rg_dist (+pbase) num_epoch = num_date * 3 + num_step + 3 if inps.geom_file: geom_obj = geometry(inps.geom_file) geom_obj.open(print_msg=False) if 'bperp' in geom_obj.datasetNames: num_epoch += num_date # split in row/line direction based on the input memory limit num_box = int( np.ceil((num_epoch * length * width * 4) * 2.5 / (inps.maxMemory * 1024**3))) box_list = cluster.split_box2sub_boxes(box=(0, 0, width, length), num_split=num_box, dimension='y') # 3.2 prepare the input arguments for *_patch() data_kwargs = { 'G_defo': G_defo, 'ts_file': inps.timeseries_file, 'geom_file': inps.geom_file, 'date_flag': date_flag, 'phase_velocity': inps.phaseVelocity, } # 3.3 invert / write block-by-block for i, box in enumerate(box_list): box_wid = box[2] - box[0] box_len = box[3] - box[1] if num_box > 1: print('\n------- processing patch {} out of {} --------------'. format(i + 1, num_box)) print('box width: {}'.format(box_wid)) print('box length: {}'.format(box_len)) # update box argument in the input data data_kwargs['box'] = box # invert if not inps.cluster: # non-parallel delta_z, ts_cor, ts_res = correct_dem_error_patch( **data_kwargs)[:-1] else: # parallel print('\n\n------- start parallel processing using Dask -------') # initiate the output data delta_z = np.zeros((box_len, box_wid), dtype=np.float32) ts_cor = np.zeros((num_date, box_len, box_wid), dtype=np.float32) ts_res = np.zeros((num_date, box_len, box_wid), dtype=np.float32) # initiate dask cluster and client cluster_obj = cluster.DaskCluster(inps.cluster, inps.numWorker, config_name=inps.config) cluster_obj.open() # run dask delta_z, ts_cor, ts_res = cluster_obj.run( func=correct_dem_error_patch, func_data=data_kwargs, results=[delta_z, ts_cor, ts_res]) # close dask cluster and client cluster_obj.close() print('------- finished parallel processing -------\n\n') # write the block to disk # with 3D block in [z0, z1, y0, y1, x0, x1] # and 2D block in [y0, y1, x0, x1] # DEM error - 2D block = [box[1], box[3], box[0], box[2]] writefile.write_hdf5_block(dem_err_file, data=delta_z, datasetName='dem', block=block) # corrected time-series - 3D block = [0, num_date, box[1], box[3], box[0], box[2]] writefile.write_hdf5_block(ts_cor_file, data=ts_cor, datasetName='timeseries', block=block) # residual time-series - 3D block = [0, num_date, box[1], box[3], box[0], box[2]] writefile.write_hdf5_block(ts_res_file, data=ts_res, datasetName='timeseries', block=block) # roll back to the origial number of threads cluster.roll_back_num_threads(num_threads_dict) # time info m, s = divmod(time.time() - start_time, 60) print('time used: {:02.0f} mins {:02.1f} secs.'.format(m, s)) return dem_err_file, ts_cor_file, ts_res_file