def update_refpix_metadata(ifg_paths, refx, refy, transform, params): """ Function that adds metadata about the chosen reference pixel to each interferogram. """ pyrate_refpix_lon, pyrate_refpix_lat = mpiops.run_once(convert_pixel_value_to_geographic_coordinate, refx, refy, transform) process_ifgs_paths = mpiops.array_split(ifg_paths) for ifg_file in process_ifgs_paths: log.debug("Updating metadata for: "+ifg_file) ifg = Ifg(ifg_file) log.debug("Open dataset") ifg.open(readonly=True) nan_and_mm_convert(ifg, params) half_patch_size = params["refchipsize"] // 2 x, y = refx, refy log.debug("Extract reference pixel windows") data = ifg.phase_data[y - half_patch_size: y + half_patch_size + 1, x - half_patch_size: x + half_patch_size + 1] log.debug("Calculate standard deviation for reference window") stddev_ref_area = np.nanstd(data) log.debug("Calculate mean for reference window") mean_ref_area = np.nanmean(data) ifg.add_metadata(**{ ifc.PYRATE_REFPIX_X: str(refx), ifc.PYRATE_REFPIX_Y: str(refy), ifc.PYRATE_REFPIX_LAT: str(pyrate_refpix_lat), ifc.PYRATE_REFPIX_LON: str(pyrate_refpix_lon), ifc.PYRATE_MEAN_REF_AREA: str(mean_ref_area), ifc.PYRATE_STDDEV_REF_AREA: str(stddev_ref_area) }) ifg.write_modified_phase() ifg.close()
def pre_prepare_ifgs(ifg_paths, params): """ nan and mm convert ifgs """ ifgs = [Ifg(p) for p in ifg_paths] for i in ifgs: i.open(readonly=False) nan_and_mm_convert(i, params) return ifgs
def _write_dem_errors(ifg_paths: list, params: dict, preread_ifgs: dict) -> None: """ Convenience function for writing DEM error (one file) and DEM error correction for each IFG to disc :param ifg_paths: List of interferogram class objects. :param params: Dictionary of PyRate configuration parameters. :param preread_ifgs: Dictionary of interferogram metadata. """ tiles = params[C.TILES] # re-assemble tiles and save into dem_error dir shape = preread_ifgs[ifg_paths[0]].shape # save dem error as geotiff file in out directory gt, md, wkt = shared.get_geotiff_header_info(ifg_paths[0]) md[ifc.DATA_TYPE] = ifc.DEM_ERROR dem_error = assemble_tiles(shape, params[C.TMPDIR], tiles, out_type='dem_error') dem_error_file = os.path.join(params[C.DEM_ERROR_DIR], 'dem_error.tif') shared.remove_file_if_exists(dem_error_file) shared.write_output_geotiff(md, gt, wkt, dem_error, dem_error_file, np.nan) # read the average bperp vals for each ifg and each tile bperp = np.empty(shape=(len(tiles), len(ifg_paths)), dtype=np.float64) for t in tiles: bperp_file = Path( join(params[C.TMPDIR], 'bperp_avg_' + str(t.index) + '.npy')) arr = np.load(file=bperp_file) bperp[t.index, :] = arr # loop over all ifgs idx = 0 for ifg_path in ifg_paths: ifg = Ifg(ifg_path) ifg.open() shared.nan_and_mm_convert(ifg, params) # ensure we have phase data in mm # read dem error correction file from tmpdir dem_error_correction_ifg = assemble_tiles( shape, params[C.TMPDIR], tiles, out_type='dem_error_correction', index=idx) # calculate average bperp value across all tiles for the ifg bperp_val = np.nanmean(bperp[:, idx]) dem_error_correction_on_disc = MultiplePaths.dem_error_path( ifg.data_path, params) np.save(file=dem_error_correction_on_disc, arr=dem_error_correction_ifg) idx += 1 # subtract DEM error from the ifg ifg.phase_data -= dem_error_correction_ifg _save_dem_error_corrected_phase(ifg, bperp_val)
def independent_orbital_correction(ifg, params): """ Calculates and removes an orbital error surface from a single independent interferogram. Warning: This will write orbital error corrected phase_data to the ifg. :param Ifg class instance ifg: the interferogram to be corrected :param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC) :param bool offset: True to calculate the model using an offset :param dict params: dictionary of configuration parameters :return: None - interferogram phase data is updated and saved to disk """ degree = params[cf.ORBITAL_FIT_DEGREE] offset = params[cf.ORBFIT_OFFSET] orbfit_correction_on_disc = MultiplePaths.orb_error_path( ifg.data_path, params) if not ifg.is_open: ifg.open() shared.nan_and_mm_convert(ifg, params) if orbfit_correction_on_disc.exists(): log.info( f'Reusing already computed orbital fit correction for {ifg.data_path}' ) orbital_correction = np.load(file=orbfit_correction_on_disc) else: # vectorise, keeping NODATA vphase = reshape(ifg.phase_data, ifg.num_cells) dm = get_design_matrix(ifg, degree, offset) # filter NaNs out before getting model clean_dm = dm[~isnan(vphase)] data = vphase[~isnan(vphase)] model = lstsq(clean_dm, data)[0] # first arg is the model params # calculate forward model & morph back to 2D if offset: fullorb = np.reshape(np.dot(dm[:, :-1], model[:-1]), ifg.phase_data.shape) else: fullorb = np.reshape(np.dot(dm, model), ifg.phase_data.shape) if not orbfit_correction_on_disc.parent.exists(): shared.mkdir_p(orbfit_correction_on_disc.parent) offset_removal = nanmedian(np.ravel(ifg.phase_data - fullorb)) orbital_correction = fullorb - offset_removal # dump to disc np.save(file=orbfit_correction_on_disc, arr=orbital_correction) # subtract orbital error from the ifg ifg.phase_data -= orbital_correction # set orbfit meta tag and save phase to file _save_orbital_error_corrected_phase(ifg) ifg.close()
def pre_prepare_ifgs(ifg_paths, params): """ Open ifg for reading """ ifgs = [Ifg(p) for p in ifg_paths] for i in ifgs: if not i.is_open: i.open(readonly=False) nan_and_mm_convert(i, params) return ifgs
def __inner(ifg, ref_ph): ifg.open() # nan-convert and mm-convert before subtracting ref phase nan_and_mm_convert(ifg, params) # add 1e-20 to avoid 0.0 values being converted to NaN downstream (Github issue #310) # TODO: implement a more robust way of avoiding this issue, e.g. using numpy masked # arrays to mark invalid pixel values rather than directly changing values to NaN ifg.phase_data -= ref_ph + 1e-20 ifg.meta_data[ifc.PYRATE_REF_PHASE] = ifc.REF_PHASE_REMOVED ifg.write_modified_phase() log.debug(f"Reference phase corrected for {ifg.data_path}") ifg.close()
def __create_multilooked_datasets(params): exts, ifg_paths, multi_paths = __extents_from_params(params) mlooked_datasets = [ _create_mlooked_dataset(m, i, exts, params) for m, i in zip(multi_paths, ifg_paths) ] mlooked = [Ifg(m) for m in mlooked_datasets] for m in mlooked: m.initialize() shared.nan_and_mm_convert(m, params) return mlooked
def cvd(ifg_path, params, r_dist, calc_alpha=False, write_vals=False, save_acg=False): """ Calculate the 1D covariance function of an entire interferogram as the radial average of its 2D autocorrelation. :param str ifg_path: An interferogram file path. OR :param dict params: Dictionary of configuration parameters :param ndarray r_dist: Array of distance values from the image centre (See Rdist class for more details) :param bool calc_alpha: If True calculate alpha :param bool write_vals: If True write maxvar and alpha values to interferogram metadata :param bool save_acg: If True write autocorrelation and radial distance data to numpy array file on disk :return: maxvar: The maximum variance (at zero lag) :rtype: float :return: alpha: the exponential length-scale of decay factor :rtype: float """ ifg = shared.Ifg(ifg_path) ifg.open() shared.nan_and_mm_convert(ifg, params) # calculate 2D auto-correlation of image using the # spectral method (Wiener-Khinchin theorem) if ifg.nan_converted: # if nancoverted earlier, convert nans back to 0's phase = where(isnan(ifg.phase_data), 0, ifg.phase_data) else: phase = ifg.phase_data maxvar, alpha = cvd_from_phase(phase, ifg, r_dist, calc_alpha, save_acg=save_acg, params=params) if write_vals: ifg.add_metadata(**{ ifc.PYRATE_MAXVAR: str(maxvar), ifc.PYRATE_ALPHA: str(alpha) }) ifg.close() return maxvar, alpha
def __plot_ifg(file, cmap, ax, num_ifgs, params): try: ifg = Ifg(file) ifg.open() except: raise AttributeError(f'Cannot open interferogram geotiff: {file}') # change nodata values to NaN for display; convert units to mm (if in radians) nan_and_mm_convert(ifg, params) im = ax.imshow(ifg.phase_data, cmap=cmap) text = ax.set_title(Path(ifg.data_path).stem) text.set_fontsize(20) # text.set_fontsize(min(20, int(num_ifgs / 2))) divider = make_axes_locatable(ax) cax = divider.append_axes("right", size="5%", pad=0.05) plt.colorbar(im, cax=cax, label='mm') ifg.close()
def __check_and_apply_orberrors_found_on_disc(ifg_paths, params): saved_orb_err_paths = [ MultiplePaths.orb_error_path(ifg_path, params) for ifg_path in ifg_paths ] for p, i in zip(saved_orb_err_paths, ifg_paths): if p.exists(): orb = np.load(p) if isinstance(i, str): # are paths ifg = Ifg(i) ifg.open(readonly=False) shared.nan_and_mm_convert(ifg, params) else: ifg = i ifg.phase_data -= orb # set orbfit meta tag and save phase to file _save_orbital_error_corrected_phase(ifg) return all(p.exists() for p in saved_orb_err_paths)
def independent_orbital_correction(ifg, degree, offset, params): """ Calculates and removes an orbital error surface from a single independent interferogram. Warning: This will write orbital error corrected phase_data to the ifg. :param Ifg class instance ifg: the interferogram to be corrected :param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC) :param bool offset: True to calculate the model using an offset :param dict params: dictionary of configuration parameters :return: None - interferogram phase data is updated and saved to disk """ ifg = shared.Ifg(ifg) if isinstance(ifg, str) else ifg if not ifg.is_open: ifg.open() shared.nan_and_mm_convert(ifg, params) # vectorise, keeping NODATA vphase = reshape(ifg.phase_data, ifg.num_cells) dm = get_design_matrix(ifg, degree, offset) # filter NaNs out before getting model clean_dm = dm[~isnan(vphase)] data = vphase[~isnan(vphase)] model = lstsq(clean_dm, data)[0] # first arg is the model params # calculate forward model & morph back to 2D if offset: fullorb = np.reshape(np.dot(dm[:, :-1], model[:-1]), ifg.phase_data.shape) else: fullorb = np.reshape(np.dot(dm, model), ifg.phase_data.shape) offset_removal = nanmedian(np.ravel(ifg.phase_data - fullorb)) # subtract orbital error from the ifg ifg.phase_data -= (fullorb - offset_removal) # set orbfit meta tag and save phase to file _save_orbital_error_corrected_phase(ifg) if ifg.open(): ifg.close()
def _apply_aps_correction(ifg_paths: List[str], aps_paths: List[str], params: dict) -> None: """ Function to read and apply (subtract) APS corrections from interferogram data. """ for ifg_path, aps_path in mpiops.array_split( list(zip(ifg_paths, aps_paths))): # read the APS correction from numpy array aps_corr = np.load(aps_path) # open the Ifg object ifg = Ifg(ifg_path) ifg.open(readonly=False) # convert NaNs and convert to mm nan_and_mm_convert(ifg, params) # subtract the correction from the ifg phase data ifg.phase_data[~np.isnan(ifg.phase_data )] -= aps_corr[~np.isnan(ifg.phase_data)] # set meta-data tags after aps error correction ifg.dataset.SetMetadataItem(ifc.PYRATE_APS_ERROR, ifc.APS_REMOVED) # write phase data to disc and close ifg. ifg.write_modified_phase() ifg.close()
def _create_ifg_dict(params): """ Save the preread_ifgs dict with information about the ifgs that are later used for fast loading of Ifg files in IfgPart class :param list dest_tifs: List of destination tifs :param dict params: Config dictionary :param list tiles: List of all Tile instances :return: preread_ifgs: Dictionary containing information regarding interferograms that are used later in workflow :rtype: dict """ dest_tifs = [ifg_path for ifg_path in params[C.INTERFEROGRAM_FILES]] ifgs_dict = {} process_tifs = mpiops.array_split(dest_tifs) for d in process_tifs: ifg = Ifg(d.tmp_sampled_path) # get the writable copy ifg.open() nan_and_mm_convert(ifg, params) ifgs_dict[d.tmp_sampled_path] = PrereadIfg( path=d.sampled_path, tmp_path=d.tmp_sampled_path, nan_fraction=ifg.nan_fraction, first=ifg.first, second=ifg.second, time_span=ifg.time_span, nrows=ifg.nrows, ncols=ifg.ncols, metadata=ifg.meta_data ) ifg.write_modified_phase() # update phase converted to mm ifg.close() ifgs_dict = join_dicts(mpiops.comm.allgather(ifgs_dict)) ifgs_dict = mpiops.run_once(__save_ifgs_dict_with_headers_and_epochs, dest_tifs, ifgs_dict, params, process_tifs) params[C.PREREAD_IFGS] = ifgs_dict return ifgs_dict
def __create_multilooked_dataset_for_network_correction(params): multi_paths = params[cf.INTERFEROGRAM_FILES] ifg_paths = [p.tmp_sampled_path for p in multi_paths] headers = [find_header(p, params) for p in multi_paths] crop_opt = prepifg_helper.ALREADY_SAME_SIZE xlooks = params[cf.ORBITAL_FIT_LOOKS_X] ylooks = params[cf.ORBITAL_FIT_LOOKS_Y] thresh = params[cf.NO_DATA_AVERAGING_THRESHOLD] rasters = [shared.dem_or_ifg(r) for r in ifg_paths] exts = prepifg_helper.get_analysis_extent(crop_opt, rasters, xlooks, ylooks, None) out_paths = [tempfile.mktemp() for _ in ifg_paths] mlooked_dataset = [ prepifg_helper.prepare_ifg(d, xlooks, ylooks, exts, thresh, crop_opt, h, False, p) for d, h, p in zip(ifg_paths, headers, out_paths) ] mlooked = [Ifg(m[1]) for m in mlooked_dataset] for m in mlooked: m.initialize() shared.nan_and_mm_convert(m, params) return mlooked
def mask_pixels_with_unwrapping_errors(ifgs_breach_count: NDArray[( Any, Any, Any), UInt16], num_occurrences_each_ifg: NDArray[(Any, ), UInt16], params: dict) -> None: """ Find pixels in the phase data that breach closure_thr, and mask (assign NaNs) to those pixels in those ifgs. :param ifgs_breach_count: unwrapping issues at pixels in all loops :param num_occurrences_each_ifg: frequency of ifgs appearing in all loops :param params: params dict """ log.debug("Masking phase data of retained ifgs") for i, m_p in enumerate(params[C.INTERFEROGRAM_FILES]): pix_index = ifgs_breach_count[:, :, i] == num_occurrences_each_ifg[i] ifg = Ifg(m_p.tmp_sampled_path) ifg.open() nan_and_mm_convert(ifg, params) ifg.phase_data[pix_index] = np.nan ifg.write_modified_phase() log.info(f"Masked phase data of {i + 1} retained ifgs after phase closure") return None
def network_orbital_correction(ifg_paths, params, m_ifgs: Optional[List] = None): """ This algorithm implements a network inversion to determine orbital corrections for a set of interferograms forming a connected network. Warning: This will write orbital error corrected phase_data to the ifgs. :param list ifg_paths: List of Ifg class objects reduced to a minimum spanning tree network :param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC) :param bool offset: True to calculate the model using offsets :param dict params: dictionary of configuration parameters :param list m_ifgs: list of multilooked Ifg class objects (sequence must be multilooked versions of 'ifgs' arg) :param dict preread_ifgs: Dictionary containing information specifically for MPI jobs (optional) :return: None - interferogram phase data is updated and saved to disk """ # pylint: disable=too-many-locals, too-many-arguments offset = params[cf.ORBFIT_OFFSET] degree = params[cf.ORBITAL_FIT_DEGREE] preread_ifgs = params[cf.PREREAD_IFGS] # all orbit corrections available? if isinstance(ifg_paths[0], str): if __check_and_apply_orberrors_found_on_disc(ifg_paths, params): log.warning("Reusing orbfit errors from previous run!!!") return # all corrections are available in numpy files already saved - return ifgs = [shared.Ifg(i) for i in ifg_paths] else: # alternate test paths # TODO: improve ifgs = ifg_paths src_ifgs = ifgs if m_ifgs is None else m_ifgs src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst vphase = vstack([i.phase_data.reshape((i.num_cells, 1)) for i in src_ifgs]) vphase = squeeze(vphase) B = get_network_design_matrix(src_ifgs, degree, offset) # filter NaNs out before getting model B = B[~isnan(vphase)] orbparams = dot(pinv(B, 1e-6), vphase[~isnan(vphase)]) ncoef = _get_num_params(degree) if preread_ifgs: temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values() ids = first_second_ids(get_all_epochs(temp_ifgs)) else: ids = first_second_ids(get_all_epochs(ifgs)) coefs = [ orbparams[i:i + ncoef] for i in range(0, len(set(ids)) * ncoef, ncoef) ] # create full res DM to expand determined coefficients into full res # orbital correction (eg. expand coarser model to full size) if preread_ifgs: temp_ifg = Ifg(ifg_paths[0]) # ifgs here are paths temp_ifg.open() dm = get_design_matrix(temp_ifg, degree, offset=False) temp_ifg.close() else: ifg = ifgs[0] dm = get_design_matrix(ifg, degree, offset=False) for i in ifg_paths: # open if not Ifg instance if isinstance(i, str): # pragma: no cover # are paths i = Ifg(i) i.open(readonly=False) shared.nan_and_mm_convert(i, params) _remove_network_orb_error(coefs, dm, i, ids, offset, params)
def independent_orbital_correction(ifg_path, params): """ Calculates and removes an orbital error surface from a single independent interferogram. Warning: This will write orbital error corrected phase_data to the ifg. :param Ifg class instance ifg: the interferogram to be corrected :param dict params: dictionary of configuration parameters :return: None - interferogram phase data is updated and saved to disk """ log.debug(f"Orbital correction of {ifg_path}") degree = params[C.ORBITAL_FIT_DEGREE] offset = params[C.ORBFIT_OFFSET] intercept = params[C.ORBFIT_INTERCEPT] xlooks = params[C.ORBITAL_FIT_LOOKS_X] ylooks = params[C.ORBITAL_FIT_LOOKS_Y] ifg0 = shared.Ifg(ifg_path) if isinstance(ifg_path, str) else ifg_path # get full-resolution design matrix fullres_dm = get_design_matrix(ifg0, degree, intercept=intercept) ifg = shared.dem_or_ifg(ifg_path) if isinstance(ifg_path, str) else ifg_path multi_path = MultiplePaths(ifg.data_path, params) orb_on_disc = MultiplePaths.orb_error_path(ifg.data_path, params) if not ifg.is_open: ifg.open() shared.nan_and_mm_convert(ifg, params) fullres_ifg = ifg # keep a backup fullres_phase = fullres_ifg.phase_data if orb_on_disc.exists(): log.info( f'Reusing already computed orbital fit correction: {orb_on_disc}') orbital_correction = np.load(file=orb_on_disc) else: # Multi-look the ifg data if either X or Y is greater than 1 if (xlooks > 1) or (ylooks > 1): exts, _, _ = __extents_from_params(params) mlooked = _create_mlooked_dataset(multi_path, ifg.data_path, exts, params) ifg = Ifg(mlooked) # multi-looked Ifg object ifg.initialize() shared.nan_and_mm_convert(ifg, params) # vectorise phase data, keeping NODATA vphase = reshape(ifg.phase_data, ifg.num_cells) # compute design matrix for multi-looked data mlooked_dm = get_design_matrix(ifg, degree, intercept=intercept) # invert to obtain the correction image (forward model) at full-res orbital_correction = __orb_correction(fullres_dm, mlooked_dm, fullres_phase, vphase, offset=offset) # save correction to disc if not orb_on_disc.parent.exists(): shared.mkdir_p(orb_on_disc.parent) np.save(file=orb_on_disc, arr=orbital_correction) # subtract orbital correction from the full-res ifg fullres_ifg.phase_data -= orbital_correction # set orbfit meta tag and save phase to file _save_orbital_error_corrected_phase(fullres_ifg, params)
def network_orbital_correction(ifg_paths, params, m_ifgs: Optional[List] = None): """ This algorithm implements a network inversion to determine orbital corrections for a set of interferograms forming a connected network. Warning: This will write orbital error corrected phase_data to the ifgs. :param list ifg_paths: List of Ifg class objects reduced to a minimum spanning tree network :param dict params: dictionary of configuration parameters :param list m_ifgs: list of multilooked Ifg class objects (sequence must be multilooked versions of 'ifgs' arg) :return: None - interferogram phase data is updated and saved to disk """ # pylint: disable=too-many-locals, too-many-arguments offset = params[C.ORBFIT_OFFSET] degree = params[C.ORBITAL_FIT_DEGREE] preread_ifgs = params[C.PREREAD_IFGS] intercept = params[C.ORBFIT_INTERCEPT] scale = params[C.ORBFIT_SCALE] # all orbit corrections available? if isinstance(ifg_paths[0], str): if __check_and_apply_orberrors_found_on_disc(ifg_paths, params): log.warning("Reusing orbfit errors from previous run!!!") return # all corrections are available in numpy files already saved - return ifgs = [shared.Ifg(i) for i in ifg_paths] else: # alternate test paths # TODO: improve ifgs = ifg_paths src_ifgs = ifgs if m_ifgs is None else m_ifgs src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst if preread_ifgs: temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values() ids = first_second_ids(get_all_epochs(temp_ifgs)) else: ids = first_second_ids(get_all_epochs(ifgs)) nepochs = len(set(ids)) # call the actual inversion routine coefs = calc_network_orb_correction(src_ifgs, degree, scale, nepochs, intercept=intercept) # create full res DM to expand determined coefficients into full res # orbital correction (eg. expand coarser model to full size) if preread_ifgs: temp_ifg = Ifg(ifg_paths[0]) # ifgs here are paths temp_ifg.open() dm = get_design_matrix(temp_ifg, degree, intercept=intercept, scale=scale) temp_ifg.close() else: ifg = ifgs[0] dm = get_design_matrix(ifg, degree, intercept=intercept, scale=scale) for i in ifg_paths: # open if not Ifg instance if isinstance(i, str): # pragma: no cover # are paths i = Ifg(i) i.open(readonly=False) shared.nan_and_mm_convert(i, params) _remove_network_orb_error(coefs, dm, i, ids, offset, params)