def maxvar_vcm_calc_wrapper(params): """ MPI wrapper for maxvar and vcmt computation """ preread_ifgs = params[cf.PREREAD_IFGS] ifg_paths = [ifg_path.tmp_sampled_path for ifg_path in params[cf.INTERFEROGRAM_FILES]] log.info('Calculating the temporal variance-covariance matrix') def _get_r_dist(ifg_path): """ Get RDIst class object """ ifg = Ifg(ifg_path) ifg.open() r_dist = RDist(ifg)() ifg.close() return r_dist r_dist = mpiops.run_once(_get_r_dist, ifg_paths[0]) prcs_ifgs = mpiops.array_split(list(enumerate(ifg_paths))) process_maxvar = {} for n, i in prcs_ifgs: log.debug(f'Calculating maxvar for {n} of process ifgs {len(prcs_ifgs)} of total {len(ifg_paths)}') process_maxvar[int(n)] = cvd(i, params, r_dist, calc_alpha=True, write_vals=True, save_acg=True)[0] maxvar_d = shared.join_dicts(mpiops.comm.allgather(process_maxvar)) maxvar = [v[1] for v in sorted(maxvar_d.items(), key=lambda s: s[0])] vcmt = mpiops.run_once(get_vcmt, preread_ifgs, maxvar) log.debug("Finished maxvar and vcm calc!") params[cf.MAXVAR], params[cf.VCMT] = maxvar, vcmt np.save(Configuration.vcmt_path(params), arr=vcmt) return maxvar, vcmt
def test_vcm_legacy_vs_mpi(mpisync, tempdir, roipac_or_gamma_conf): params = configuration.Configuration(roipac_or_gamma_conf).__dict__ LEGACY_VCM_DIR = os.path.join(SML_TEST_DIR, 'vcm') legacy_vcm = np.genfromtxt(os.path.join(LEGACY_VCM_DIR, 'vcmt.csv'), delimiter=',') tmpdir = Path(mpiops.run_once(tempdir)) mpiops.run_once(common.copytree, params[cf.OBS_DIR], tmpdir) params[cf.OUT_DIR] = tmpdir.joinpath('out') params[cf.PARALLEL] = 0 output_conf = Path(tmpdir).joinpath('conf.cfg') cf.write_config_file(params=params, output_conf_file=output_conf) params = configuration.Configuration(output_conf).__dict__ dest_paths = [p.sampled_path for p in params[cf.INTERFEROGRAM_FILES]] # run conv2tif and prepifg, create the dest_paths files conv2tif.main(params) params[cf.INTERFEROGRAM_FILES].pop() prepifg.main(params) params[cf.INTERFEROGRAM_FILES].pop() preread_ifgs = process._create_ifg_dict(dest_paths, params=params) refpx, refpy = process._ref_pixel_calc(dest_paths, params) process._orb_fit_calc(params[cf.INTERFEROGRAM_FILES], params) process._ref_phase_estimation(dest_paths, params, refpx, refpy) maxvar, vcmt = process._maxvar_vcm_calc(dest_paths, params, preread_ifgs) # phase data after ref pixel has changed due to commit bf2f7ebd # Legacy tests won't match anymore np.testing.assert_array_almost_equal(maxvar, legacy_maxvar, decimal=4) np.testing.assert_array_almost_equal(legacy_vcm, vcmt, decimal=3) mpiops.run_once(shutil.rmtree, tmpdir)
def test_vcm_legacy_vs_mpi(mpisync, tempdir, roipac_or_gamma_conf): params = configuration.Configuration(roipac_or_gamma_conf).__dict__ LEGACY_VCM_DIR = os.path.join(SML_TEST_DIR, 'vcm') legacy_vcm = np.genfromtxt(os.path.join(LEGACY_VCM_DIR, 'vcmt.csv'), delimiter=',') tmpdir = Path(mpiops.run_once(tempdir)) mpiops.run_once(common.copytree, params[cf.OBS_DIR], tmpdir) params[cf.OUT_DIR] = tmpdir.joinpath('out') params[cf.PARALLEL] = False xlks, ylks, crop = cf.transform_params(params) base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST], params[cf.OBS_DIR]) # dest_paths are tifs that have been geotif converted and multilooked dest_paths = cf.get_dest_paths(base_unw_paths, crop, params, xlks) # run conv2tif and prepifg, create the dest_paths files conv2tif.main(params) prepifg.main(params) tiles = pyrate.core.shared.get_tiles(dest_paths[0], rows=1, cols=1) preread_ifgs = process._create_ifg_dict(dest_paths, params=params, tiles=tiles) refpx, refpy = process._ref_pixel_calc(dest_paths, params) process._orb_fit_calc(dest_paths, params) process._ref_phase_estimation(dest_paths, params, refpx, refpy) maxvar, vcmt = process._maxvar_vcm_calc(dest_paths, params, preread_ifgs) np.testing.assert_array_almost_equal(maxvar, legacy_maxvar, decimal=4) np.testing.assert_array_almost_equal(legacy_vcm, vcmt, decimal=3) mpiops.run_once(shutil.rmtree, tmpdir)
def test_vcm_legacy_vs_mpi(mpisync, tempdir, roipac_or_gamma_conf): params = configuration.Configuration(roipac_or_gamma_conf).__dict__ LEGACY_VCM_DIR = os.path.join(SML_TEST_DIR, 'vcm') legacy_vcm = np.genfromtxt(os.path.join(LEGACY_VCM_DIR, 'vcmt.csv'), delimiter=',') tmpdir = Path(mpiops.run_once(tempdir)) params[C.OUT_DIR] = tmpdir.joinpath('out') params[C.PARALLEL] = 0 output_conf = Path(tmpdir).joinpath('conf.cfg') pyrate.configuration.write_config_file(params=params, output_conf_file=output_conf) params = configuration.Configuration(output_conf).__dict__ # dest_paths = [p.sampled_path for p in params[cf.INTERFEROGRAM_FILES]] # run conv2tif and prepifg, create the dest_paths files conv2tif.main(params) params = configuration.Configuration(output_conf).__dict__ prepifg.main(params) params = configuration.Configuration(output_conf).__dict__ params[C.ORBFIT_OFFSET] = True correct._copy_mlooked(params=params) correct._update_params_with_tiles(params) correct._create_ifg_dict(params=params) pyrate.core.refpixel.ref_pixel_calc_wrapper(params) pyrate.core.orbital.orb_fit_calc_wrapper(params) pyrate.core.ref_phs_est.ref_phase_est_wrapper(params) maxvar, vcmt = pyrate.core.covariance.maxvar_vcm_calc_wrapper(params) # phase data after ref pixel has changed due to commit bf2f7ebd # Legacy tests won't match anymore np.testing.assert_array_almost_equal(maxvar, legacy_maxvar, decimal=4) np.testing.assert_array_almost_equal(legacy_vcm, vcmt, decimal=3) mpiops.run_once(shutil.rmtree, tmpdir)
def spatio_temporal_filter(params: dict) -> None: """ Applies a spatio-temporal filter to remove the atmospheric phase screen (APS) and saves the corrected interferograms. Firstly the incremental time series is computed using the SVD method, before a cascade of temporal then spatial Gaussian filters is applied. The resulting APS corrections are saved to disc before being subtracted from each interferogram. :param params: Dictionary of PyRate configuration parameters. """ if params[C.APSEST]: log.info('Doing APS spatio-temporal filtering') else: log.info('APS spatio-temporal filtering not required') return tiles = params[C.TILES] preread_ifgs = params[C.PREREAD_IFGS] ifg_paths = [ ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES] ] # perform some checks on existing ifgs log.debug('Checking APS correction status') if mpiops.run_once(shared.check_correction_status, ifg_paths, ifc.PYRATE_APS_ERROR): log.debug('Finished APS correction') return # return if True condition returned aps_paths = [MultiplePaths.aps_error_path(i, params) for i in ifg_paths] if all(a.exists() for a in aps_paths): log.warning('Reusing APS errors from previous run') _apply_aps_correction(ifg_paths, aps_paths, params) return # obtain the incremental time series using SVD tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles) mpiops.comm.barrier() # get lists of epochs and ifgs ifgs = list(OrderedDict(sorted(preread_ifgs.items())).values()) epochlist = mpiops.run_once(get_epochs, ifgs)[0] # first perform temporal high pass filter ts_hp = temporal_high_pass_filter(tsincr, epochlist, params) # second perform spatial low pass filter to obtain APS correction in ts domain ifg = Ifg(ifg_paths[0]) # just grab any for parameters in slpfilter ifg.open() ts_aps = spatial_low_pass_filter(ts_hp, ifg, params) ifg.close() # construct APS corrections for each ifg _make_aps_corrections(ts_aps, ifgs, params) # apply correction to ifgs and save ifgs to disc. _apply_aps_correction(ifg_paths, aps_paths, params) # update/save the phase_data in the tiled numpy files shared.save_numpy_phase(ifg_paths, params)
def __wrap_closure_check(config: Configuration) -> \ Tuple[ List[str], NDArray[(Any, Any), Float32], NDArray[(Any, Any, Any), UInt16], NDArray[(Any,), UInt16], List[WeightedLoop]]: """ This wrapper function returns the closure check outputs for a single iteration of closure check. :param config: Configuration class instance For return variables see docstring in `sum_phase_closures`. """ params = config.__dict__ ifg_files = [ ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES] ] ifg_files.sort() log.debug(f"The number of ifgs in the list is {len(ifg_files)}") sorted_signed_loops = mpiops.run_once(sort_loops_based_on_weights_and_date, params) log.info( f"Total number of selected closed loops with up to MAX_LOOP_LENGTH = " f"{params[C.MAX_LOOP_LENGTH]} edges is {len(sorted_signed_loops)}") if len(sorted_signed_loops) < 1: return None retained_loops = mpiops.run_once(discard_loops_containing_max_ifg_count, sorted_signed_loops, params) ifgs_with_loops = mpiops.run_once(__drop_ifgs_if_not_part_of_any_loop, ifg_files, retained_loops, params) msg = f"After applying MAX_LOOP_REDUNDANCY = {params[C.MAX_LOOP_REDUNDANCY]} criteria, " \ f"{len(retained_loops)} loops are retained" if len(retained_loops) < 1: return None else: log.info(msg) closure, ifgs_breach_count, num_occurences_each_ifg = sum_phase_closures( ifgs_with_loops, retained_loops, params) if mpiops.rank == 0: closure_ins = config.closure() np.save(closure_ins.closure, closure) np.save(closure_ins.ifgs_breach_count, ifgs_breach_count) np.save(closure_ins.num_occurences_each_ifg, num_occurences_each_ifg) np.save(closure_ins.loops, retained_loops, allow_pickle=True) selected_ifg_files = mpiops.run_once(__drop_ifgs_exceeding_threshold, ifgs_with_loops, ifgs_breach_count, num_occurences_each_ifg, params) # update the ifg list in the parameters dictionary params[C.INTERFEROGRAM_FILES] = \ mpiops.run_once(update_ifg_list, selected_ifg_files, params[C.INTERFEROGRAM_FILES]) return selected_ifg_files, closure, ifgs_breach_count, num_occurences_each_ifg, retained_loops
def dem_error_calc_wrapper(params: dict) -> None: """ MPI wrapper for DEM error correction :param params: Dictionary of PyRate configuration parameters. """ if not params[C.DEMERROR]: log.info("DEM error correction not required") return # geometry information needed to calculate Bperp for each pixel using first IFG in list ifg_paths = [ ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES] ] # check if DEM error correction is already available if mpiops.run_once(__check_and_apply_demerrors_found_on_disc, ifg_paths, params): log.warning("Reusing DEM error correction from previous run!!!") else: log.info("Calculating DEM error correction") # read and open the first IFG in list ifg0_path = ifg_paths[0] ifg0 = Ifg(ifg0_path) ifg0.open(readonly=True) # not currently implemented for ROIPAC data which breaks some tests # if statement can be deleted once ROIPAC is deprecated from PyRate if ifg0.meta_data[ifc.PYRATE_INSAR_PROCESSOR] == 'ROIPAC': log.warning( "Geometry calculations are not implemented for ROI_PAC") return if params[C.BASE_FILE_LIST] is None: log.warning( "No baseline files supplied: DEM error has not been computed") return # todo: subtract other corrections (e.g. orbital) from displacement phase before estimating the DEM error # the following code is a quick way to do the bperp calculation, but is not identical to the GAMMA output # where the near range of the first SLC is used for each pair. # calculate look angle for interferograms (using the Near Range of the first SLC) # look_angle = geometry.calc_local_geometry(ifg0, None, rg, lon, lat, params) # bperp = geometry.calc_local_baseline(ifg0, az, look_angle) # split full arrays in to tiles for parallel processing tiles_split(_process_dem_error_per_tile, params) preread_ifgs = params[C.PREREAD_IFGS] # write dem error and correction values to file mpiops.run_once(_write_dem_errors, ifg_paths, params, preread_ifgs) shared.save_numpy_phase(ifg_paths, params) log.debug('Finished DEM error correction step')
def main(params): """ PyRate merge main function. Assembles product tiles in to single geotiff files """ # setup paths rows, cols = params["rows"], params["cols"] mpiops.run_once(_merge_stack, rows, cols, params) mpiops.run_once(_create_png_from_tif, params[cf.OUT_DIR]) if params[cf.TIME_SERIES_CAL]: _merge_timeseries(rows, cols, params)
def _maxvar_vcm_calc(ifg_paths, params, preread_ifgs): """ MPI wrapper for maxvar and vcmt computation """ log.info('Calculating the temporal variance-covariance matrix') process_indices = mpiops.array_split(range(len(ifg_paths))) def _get_r_dist(ifg_path): """ Get RDIst class object """ ifg = Ifg(ifg_path) ifg.open() r_dist = vcm_module.RDist(ifg)() ifg.close() return r_dist r_dist = mpiops.run_once(_get_r_dist, ifg_paths[0]) prcs_ifgs = mpiops.array_split(ifg_paths) process_maxvar = [] for n, i in enumerate(prcs_ifgs): log.debug( 'Calculating maxvar for {} of process ifgs {} of total {}'.format( n + 1, len(prcs_ifgs), len(ifg_paths))) process_maxvar.append( vcm_module.cvd(i, params, r_dist, calc_alpha=True, write_vals=True, save_acg=True)[0]) if mpiops.rank == MASTER_PROCESS: maxvar = np.empty(len(ifg_paths), dtype=np.float64) maxvar[process_indices] = process_maxvar for i in range(1, mpiops.size): # pragma: no cover rank_indices = mpiops.array_split(range(len(ifg_paths)), i) this_process_ref_phs = np.empty(len(rank_indices), dtype=np.float64) mpiops.comm.Recv(this_process_ref_phs, source=i, tag=i) maxvar[rank_indices] = this_process_ref_phs else: # pragma: no cover maxvar = np.empty(len(ifg_paths), dtype=np.float64) mpiops.comm.Send(np.array(process_maxvar, dtype=np.float64), dest=MASTER_PROCESS, tag=mpiops.rank) mpiops.comm.barrier() maxvar = mpiops.comm.bcast(maxvar, root=0) vcmt = mpiops.run_once(vcm_module.get_vcmt, preread_ifgs, maxvar) log.debug("Finished maxvar and vcm calc!") return maxvar, vcmt
def main(params: dict) -> None: """ PyRate merge main function. Assembles product tiles in to single geotiff files """ if params[C.SIGNAL_POLARITY] == 1: log.info( f"Saving output products with the same sign convention as input data" ) elif params[C.SIGNAL_POLARITY] == -1: log.info( f"Saving output products with reversed sign convention compared to the input data" ) else: log.warning(f"Check the value of signal_polarity parameter") out_types = [] tsfile = join(params[C.TMPDIR], 'tscuml_0.npy') if exists(tsfile): _merge_timeseries(params, 'tscuml') _merge_linrate(params) out_types += ['linear_rate', 'linear_error', 'linear_rsquared'] # optional save of merged tsincr products if params["savetsincr"] == 1: _merge_timeseries(params, 'tsincr') else: log.warning( 'Not merging time series products; {} does not exist'.format( tsfile)) stfile = join(params[C.TMPDIR], 'stack_rate_0.npy') if exists(stfile): # setup paths mpiops.run_once(_merge_stack, params) out_types += ['stack_rate', 'stack_error'] else: log.warning( 'Not merging stack products; {} does not exist'.format(stfile)) if len(out_types) > 0: process_out_types = mpiops.array_split(out_types) for out_type in process_out_types: create_png_and_kml_from_tif(params[C.VELOCITY_DIR], output_type=out_type) else: log.warning('Exiting: no products to merge')
def _merge_linrate(params: dict) -> None: """ Merge linear rate outputs """ shape, tiles, ifgs_dict = mpiops.run_once(__merge_setup, params) log.info('Merging and writing Linear Rate product geotiffs') # read and assemble tile outputs out_types = [ 'linear_' + x for x in ['rate', 'rsquared', 'error', 'intercept', 'samples'] ] process_out_types = mpiops.array_split(out_types) for p_out_type in process_out_types: out = assemble_tiles(shape, params[C.TMPDIR], tiles, out_type=p_out_type) __save_merged_files(ifgs_dict, params, out, p_out_type, savenpy=params["savenpy"]) mpiops.comm.barrier()
def spatio_temporal_filter(tsincr, ifg_paths, params, preread_ifgs): """ Applies a spatio-temporal filter to remove the atmospheric phase screen (APS) and saves the corrected interferograms. Before performing this step, the time series is computed using the SVD method. This function then performs temporal and spatial filtering. :param ndarray tsincr: incremental time series array of size (ifg.shape, nepochs-1) :param list ifg: List of pyrate.shared.Ifg class objects. :param dict params: Dictionary of configuration parameter :param list tiles: List of pyrate.shared.Tile class objects :param dict preread_ifgs: Dictionary of shared.PrereadIfg class instances :return: None, corrected interferograms are saved to disk """ ifg = Ifg(ifg_paths[0]) # just grab any for parameters in slpfilter ifg.open() epochlist = mpiops.run_once(get_epochs, preread_ifgs)[0] ts_lp = temporal_low_pass_filter(tsincr, epochlist, params) ts_hp = tsincr - ts_lp ts_aps = spatial_low_pass_filter(ts_hp, ifg, params) tsincr -= ts_aps _ts_to_ifgs(tsincr, preread_ifgs, params) ifg.close()
def _ts_to_ifgs(tsincr, preread_ifgs, params): """ Function that converts an incremental displacement time series into interferometric phase observations. Used to re-construct an interferogram network from a time series. :param ndarray tsincr: incremental time series array of size (ifg.shape, nepochs-1) :param dict preread_ifgs: Dictionary of shared.PrereadIfg class instances :return: None, interferograms are saved to disk """ log.debug('Reconstructing interferometric observations from time series') ifgs = list(OrderedDict(sorted(preread_ifgs.items())).values()) _, n = mpiops.run_once(get_epochs, ifgs) index_first, index_second = n[:len(ifgs)], n[len(ifgs):] num_ifgs_tuples = mpiops.array_split(list(enumerate(ifgs))) num_ifgs_tuples = [(int(num), ifg) for num, ifg in num_ifgs_tuples] for i, ifg in num_ifgs_tuples: aps_correction_on_disc = MultiplePaths.aps_error_path( ifg.tmp_path, params) phase = np.sum(tsincr[:, :, index_first[i]:index_second[i]], axis=2) np.save(file=aps_correction_on_disc, arr=phase) _save_aps_corrected_phase(ifg.tmp_path, phase)
def update_refpix_metadata(ifg_paths, refx, refy, transform, params): """ Function that adds metadata about the chosen reference pixel to each interferogram. """ pyrate_refpix_lon, pyrate_refpix_lat = mpiops.run_once(convert_pixel_value_to_geographic_coordinate, refx, refy, transform) process_ifgs_paths = mpiops.array_split(ifg_paths) for ifg_file in process_ifgs_paths: log.debug("Updating metadata for: "+ifg_file) ifg = Ifg(ifg_file) log.debug("Open dataset") ifg.open(readonly=True) nan_and_mm_convert(ifg, params) half_patch_size = params["refchipsize"] // 2 x, y = refx, refy log.debug("Extract reference pixel windows") data = ifg.phase_data[y - half_patch_size: y + half_patch_size + 1, x - half_patch_size: x + half_patch_size + 1] log.debug("Calculate standard deviation for reference window") stddev_ref_area = np.nanstd(data) log.debug("Calculate mean for reference window") mean_ref_area = np.nanmean(data) ifg.add_metadata(**{ ifc.PYRATE_REFPIX_X: str(refx), ifc.PYRATE_REFPIX_Y: str(refy), ifc.PYRATE_REFPIX_LAT: str(pyrate_refpix_lat), ifc.PYRATE_REFPIX_LON: str(pyrate_refpix_lon), ifc.PYRATE_MEAN_REF_AREA: str(mean_ref_area), ifc.PYRATE_STDDEV_REF_AREA: str(stddev_ref_area) }) ifg.write_modified_phase() ifg.close()
def _ref_pixel_calc(ifg_paths, params): """ Wrapper for reference pixel calculation """ refx = params[cf.REFX] refy = params[cf.REFY] ifg = Ifg(ifg_paths[0]) ifg.open(readonly=True) if refx == -1 or refy == -1: log.info('Searching for best reference pixel location') half_patch_size, thresh, grid = refpixel.ref_pixel_setup( ifg_paths, params) process_grid = mpiops.array_split(grid) refpixel.save_ref_pixel_blocks(process_grid, half_patch_size, ifg_paths, params) mean_sds = refpixel._ref_pixel_mpi(process_grid, half_patch_size, ifg_paths, thresh, params) mean_sds = mpiops.comm.gather(mean_sds, root=0) if mpiops.rank == MASTER_PROCESS: mean_sds = np.hstack(mean_sds) refy, refx = mpiops.run_once(refpixel.find_min_mean, mean_sds, grid) log.info('Selected reference pixel coordinate: ({}, {})'.format( refx, refy)) else: log.info('Reusing reference pixel from config file: ({}, {})'.format( refx, refy)) ifg.close() return refx, refy
def main(params): """ PyRate merge main function. Assembles product tiles in to single geotiff files """ # setup paths rows, cols = params["rows"], params["cols"] _merge_stack(rows, cols, params) if params[cf.TIME_SERIES_CAL]: _merge_timeseries(rows, cols, params) #mpiops.run_once(_delete_tsincr_files, params) log.info('Creating quicklook images.') mpiops.run_once(create_png_from_tif, params[cf.OUT_DIR]) log.debug('Finished creating quicklook images.')
def _orb_fit_calc(ifg_paths, params, preread_ifgs=None): """ MPI wrapper for orbital fit correction """ log.info('Calculating orbital correction') if not params[cf.ORBITAL_FIT]: log.info('Orbital correction not required') return if preread_ifgs: # don't check except for mpi tests # perform some general error/sanity checks log.debug('Checking Orbital error correction status') if mpiops.run_once(shared.check_correction_status, ifg_paths, ifc.PYRATE_ORBITAL_ERROR): log.debug('Finished Orbital error correction') return # return if True condition returned if params[cf.ORBITAL_FIT_METHOD] == 1: prcs_ifgs = mpiops.array_split(ifg_paths) orbital.remove_orbital_error(prcs_ifgs, params, preread_ifgs) else: # Here we do all the multilooking in one process, but in memory # can use multiple processes if we write data to disc during # remove_orbital_error step # A performance comparison should be made for saving multilooked # files on disc vs in memory single process multilooking if mpiops.rank == MASTER_PROCESS: orbital.remove_orbital_error(ifg_paths, params, preread_ifgs) mpiops.comm.barrier() log.debug('Finished Orbital error correction')
def _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles): """ Helper function to obtain time series for spatio-temporal filter using SVD method """ # Is there other existing functions that can perform this same job? log.info('Calculating time series via SVD method for ' 'APS correction') # copy params temporarily new_params = deepcopy(params) new_params[cf.TIME_SERIES_METHOD] = 2 # use SVD method process_tiles = mpiops.array_split(tiles) nvels = None for t in process_tiles: log.debug('Calculating time series for tile {} during APS ' 'correction'.format(t.index)) ifg_parts = [ shared.IfgPart(p, t, preread_ifgs, params) for p in ifg_paths ] mst_tile = np.load( os.path.join(params[cf.TMPDIR], 'mst_mat_{}.npy'.format(t.index))) tsincr = time_series(ifg_parts, new_params, vcmt=None, mst=mst_tile)[0] np.save(file=os.path.join(params[cf.TMPDIR], 'tsincr_aps_{}.npy'.format(t.index)), arr=tsincr) nvels = tsincr.shape[2] nvels = mpiops.comm.bcast(nvels, root=0) mpiops.comm.barrier() # need to assemble tsincr from all processes tsincr_g = mpiops.run_once(_assemble_tsincr, ifg_paths, params, preread_ifgs, tiles, nvels) log.debug('Finished calculating time series for spatio-temporal filter') return tsincr_g
def wrap_spatio_temporal_filter(ifg_paths, params, tiles, preread_ifgs): """ A wrapper for the spatio-temporal filter so it can be tested. See docstring for spatio_temporal_filter. """ if params[cf.APSEST]: log.info('Doing APS spatio-temporal filtering') else: log.info('APS spatio-temporal filtering not required') return # perform some checks on existing ifgs log.debug('Checking APS correction status') if mpiops.run_once(shared.check_correction_status, ifg_paths, ifc.PYRATE_APS_ERROR): log.debug('Finished APS correction') return # return if True condition returned tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles) mpiops.comm.barrier() ifg = Ifg(ifg_paths[0]) # just grab any for parameters in slpfilter ifg.open() spatio_temporal_filter(tsincr, ifg, params, preread_ifgs) ifg.close() mpiops.comm.barrier()
def process_ifgs(ifg_paths, params, rows, cols): """ Top level function to perform PyRate workflow on given interferograms :param list ifg_paths: List of interferogram paths :param dict params: Dictionary of configuration parameters :param int rows: Number of sub-tiles in y direction :param int cols: Number of sub-tiles in x direction :return: refpt: tuple of reference pixel x and y position :rtype: tuple :return: maxvar: array of maximum variance values of interferograms :rtype: ndarray :return: vcmt: Variance-covariance matrix array :rtype: ndarray """ if mpiops.size > 1: # turn of multiprocessing during mpi jobs params[cf.PARALLEL] = False outdir = params[cf.TMPDIR] if not os.path.exists(outdir): shared.mkdir_p(outdir) tiles = mpiops.run_once(get_tiles, ifg_paths[0], rows, cols) preread_ifgs = _create_ifg_dict(ifg_paths, params=params) # validate user supplied ref pixel refpixel.validate_supplied_lat_lon(params) refpx, refpy = _ref_pixel_calc(ifg_paths, params) # remove non ifg keys _ = [preread_ifgs.pop(k) for k in ['gt', 'epochlist', 'md', 'wkt']] multi_paths = params[cf.INTERFEROGRAM_FILES] _orb_fit_calc(multi_paths, params, preread_ifgs) _ref_phase_estimation(ifg_paths, params, refpx, refpy) shared.save_numpy_phase(ifg_paths, tiles, params) _mst_calc(ifg_paths, params, tiles, preread_ifgs) # spatio-temporal aps filter wrap_spatio_temporal_filter(ifg_paths, params, tiles, preread_ifgs) maxvar, vcmt = _maxvar_vcm_calc(ifg_paths, params, preread_ifgs) # save phase data tiles as numpy array for timeseries and stackrate calc shared.save_numpy_phase(ifg_paths, tiles, params) _timeseries_calc(ifg_paths, params, vcmt, tiles, preread_ifgs) _stack_calc(ifg_paths, params, vcmt, tiles, preread_ifgs) log.info('PyRate workflow completed') return (refpx, refpy), maxvar, vcmt
def remove_orbital_error(ifgs: List, params: dict) -> None: """ Wrapper function for PyRate orbital error removal functionality. NB: the ifg data is modified in situ, rather than create intermediate files. The network method assumes the given ifgs have already been reduced to a minimum spanning tree network. """ mpiops.run_once(__orb_params_check, params) ifg_paths = [i.data_path for i in ifgs] if isinstance(ifgs[0], Ifg) else ifgs method = params[cf.ORBITAL_FIT_METHOD] # mlooking is not necessary for independent correction in a computational sense # can use multiple procesing if write_to_disc=True if method == INDEPENDENT_METHOD: log.info('Calculating orbital correction using independent method') #TODO: implement multi-looking for independent orbit method if params[cf.ORBITAL_FIT_LOOKS_X] > 1 or params[ cf.ORBITAL_FIT_LOOKS_Y] > 1: log.warning( 'Multi-looking is not applied in independent orbit method') ifgs = [shared.Ifg(p) for p in ifg_paths] if isinstance(ifgs[0], str) else ifgs process_ifgs = mpiops.array_split(ifgs) for ifg in process_ifgs: independent_orbital_correction(ifg, params=params) elif method == NETWORK_METHOD: log.info('Calculating orbital correction using network method') # Here we do all the multilooking in one process, but in memory # can use multiple processes if we write data to disc during # remove_orbital_error step # A performance comparison should be made for saving multilooked # files on disc vs in memory single process multilooking if mpiops.rank == MAIN_PROCESS: mlooked = __create_multilooked_dataset_for_network_correction( params) _validate_mlooked(mlooked, ifg_paths) network_orbital_correction(ifg_paths, params, mlooked) else: raise OrbitalError("Unrecognised orbital correction method")
def test_prepifg_mpi(mpisync, get_config, tempdir, roipac_or_gamma, get_lks, get_crop): from tests.common import TEST_CONF_ROIPAC, TEST_CONF_GAMMA from os.path import join, basename if roipac_or_gamma == 1: params = get_config(TEST_CONF_GAMMA) else: params = get_config(TEST_CONF_ROIPAC) outdir = mpiops.run_once(tempdir) params[cf.OUT_DIR] = outdir params[cf.PARALLEL] = False params[cf.IFG_LKSX], params[cf.IFG_LKSY] = get_lks, get_lks params[cf.IFG_CROP_OPT] = get_crop if roipac_or_gamma == 1: params[cf.IFG_FILE_LIST] = join(common.SML_TEST_GAMMA, 'ifms_17') params[cf.OBS_DIR] = common.SML_TEST_GAMMA params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA params[cf.DEM_HEADER_FILE] = common.SML_TEST_DEM_HDR_GAMMA conv2tif.main(params) prepifg.main(params) common.remove_tifs(params[cf.OBS_DIR]) if mpiops.rank == 0: if roipac_or_gamma == 1: params_s = get_config(TEST_CONF_GAMMA) else: params_s = get_config(TEST_CONF_ROIPAC) params_s[cf.OUT_DIR] = tempdir() params_s[cf.PARALLEL] = True params_s[cf.IFG_LKSX], params_s[cf.IFG_LKSY] = get_lks, get_lks params_s[cf.IFG_CROP_OPT] = get_crop conv2tif.main(params) if roipac_or_gamma == 1: base_unw_paths = glob.glob(join(common.SML_TEST_GAMMA, "*_utm.unw")) prepifg.main(params) else: base_unw_paths = glob.glob(join(common.SML_TEST_OBS, "*.unw")) prepifg.main(params_s) mpi_tifs = glob.glob(join(outdir, "*.tif")) serial_tifs = glob.glob(join(params[cf.OUT_DIR], "*.tif")) mpi_tifs.sort() serial_tifs.sort() # 17 geotifs, and 17 mlooked tifs assert len(mpi_tifs) == len(serial_tifs) for m_f, s_f in zip(mpi_tifs, serial_tifs): assert basename(m_f) == basename(s_f) shutil.rmtree(outdir) shutil.rmtree(params_s[cf.OUT_DIR]) common.remove_tifs(params[cf.OBS_DIR])
def process_ifgs(ifg_paths, params, rows, cols): """ Top level function to perform PyRate workflow on given interferograms :param list ifg_paths: List of interferogram paths :param dict params: Dictionary of configuration parameters :param int rows: Number of sub-tiles in y direction :param int cols: Number of sub-tiles in x direction :return: refpt: tuple of reference pixel x and y position :rtype: tuple :return: maxvar: array of maximum variance values of interferograms :rtype: ndarray :return: vcmt: Variance-covariance matrix array :rtype: ndarray """ if mpiops.size > 1: # turn of multiprocessing during mpi jobs params[cf.PARALLEL] = False tiles = mpiops.run_once(get_tiles, ifg_paths[0], rows, cols) preread_ifgs = _create_ifg_dict(ifg_paths, params=params, tiles=tiles) # _mst_calc(ifg_paths, params, tiles, preread_ifgs) refpx, refpy = _ref_pixel_calc(ifg_paths, params) log.debug("refpx, refpy: " + str(refpx) + " " + str(refpy)) # remove non ifg keys _ = [preread_ifgs.pop(k) for k in ['gt', 'epochlist', 'md', 'wkt']] _orb_fit_calc(ifg_paths, params, preread_ifgs) _ref_phase_estimation(ifg_paths, params, refpx, refpy) _mst_calc(ifg_paths, params, tiles, preread_ifgs) # spatio-temporal aps filter _wrap_spatio_temporal_filter(ifg_paths, params, tiles, preread_ifgs) maxvar, vcmt = _maxvar_vcm_calc(ifg_paths, params, preread_ifgs) # save phase data tiles as numpy array for timeseries and linrate calc shared.save_numpy_phase(ifg_paths, tiles, params) _timeseries_calc(ifg_paths, params, vcmt, tiles, preread_ifgs) _linrate_calc(ifg_paths, params, vcmt, tiles, preread_ifgs) log.info('PyRate workflow completed') return (refpx, refpy), maxvar, vcmt
def configure_stage_log(verbosity, step_name, log_file_name='pyrate.log.'): log_file_name = run_once(str.__add__, log_file_name, step_name + '.' + datetime.now().isoformat()) ch = MPIStreamHandler() ch.setLevel(verbosity) ch.setFormatter(formatter) fh = logging.FileHandler(log_file_name) fh.setLevel(verbosity) fh.setFormatter(formatter) pyratelogger.addHandler(ch) pyratelogger.addHandler(fh)
def main(params: dict) -> None: """ PyRate merge main function. Assembles product tiles in to single geotiff files """ out_types = [] stfile = join(params[cf.TMPDIR], 'stack_rate_0.npy') if exists(stfile): # setup paths mpiops.run_once(_merge_stack, params) out_types += ['stack_rate', 'stack_error'] else: log.warning( 'Not merging stack products; {} does not exist'.format(stfile)) tsfile = join(params[cf.TMPDIR], 'tscuml_0.npy') if exists(tsfile): _merge_timeseries(params, 'tscuml') _merge_linrate(params) out_types += ['linear_rate', 'linear_error', 'linear_rsquared'] # optional save of merged tsincr products if params["savetsincr"] == 1: _merge_timeseries(params, 'tsincr') else: log.warning( 'Not merging time series products; {} does not exist'.format( tsfile)) if len(out_types) > 0: process_out_types = mpiops.array_split(out_types) for out_type in process_out_types: create_png_and_kml_from_tif(params[cf.OUT_DIR], output_type=out_type) else: log.warning('Exiting: no products to merge')
def _ref_phase_estimation(ifg_paths, params, refpx, refpy): """ Wrapper for reference phase estimation. """ log.info("Calculating reference phase and correcting each interferogram") if len(ifg_paths) < 2: raise rpe.ReferencePhaseError( "At least two interferograms required for reference phase correction ({len_ifg_paths} " "provided).".format(len_ifg_paths=len(ifg_paths))) if mpiops.run_once(shared.check_correction_status, ifg_paths, ifc.PYRATE_REF_PHASE): log.debug('Finished reference phase correction') return if params[cf.REF_EST_METHOD] == 1: ref_phs = rpe.est_ref_phase_method1(ifg_paths, params) elif params[cf.REF_EST_METHOD] == 2: ref_phs = rpe.est_ref_phase_method2(ifg_paths, params, refpx, refpy) else: raise rpe.ReferencePhaseError("No such option, use '1' or '2'.") # Save reference phase numpy arrays to disk. ref_phs_file = os.path.join(params[cf.TMPDIR], 'ref_phs.npy') if mpiops.rank == MASTER_PROCESS: collected_ref_phs = np.zeros(len(ifg_paths), dtype=np.float64) process_indices = mpiops.array_split(range(len(ifg_paths))) collected_ref_phs[process_indices] = ref_phs for r in range(1, mpiops.size): process_indices = mpiops.array_split(range(len(ifg_paths)), r) this_process_ref_phs = np.zeros(shape=len(process_indices), dtype=np.float64) mpiops.comm.Recv(this_process_ref_phs, source=r, tag=r) collected_ref_phs[process_indices] = this_process_ref_phs np.save(file=ref_phs_file, arr=collected_ref_phs) else: mpiops.comm.Send(ref_phs, dest=MASTER_PROCESS, tag=mpiops.rank) log.debug('Finished reference phase correction') # Preserve old return value so tests don't break. if isinstance(ifg_paths[0], Ifg): ifgs = ifg_paths else: ifgs = [Ifg(ifg_path) for ifg_path in ifg_paths] mpiops.comm.barrier() return ref_phs, ifgs
def phase_closure_wrapper(params: dict, config: Configuration) -> dict: """ This wrapper will run the iterative phase closure check to return a stable list of checked interferograms, and then mask pixels in interferograms that exceed the unwrapping error threshold. :param params: Dictionary of PyRate configuration parameters. :param config: Configuration class instance. :return: params: Updated dictionary of PyRate configuration parameters. """ if not params[C.PHASE_CLOSURE]: log.info("Phase closure correction is not required!") return rets = iterative_closure_check(config) if rets is None: log.info("Zero loops are returned from the iterative closure check.") log.warning("Abandoning phase closure correction without modifying the interferograms.") return ifg_files, ifgs_breach_count, num_occurences_each_ifg = rets # update params with closure checked ifg list params[C.INTERFEROGRAM_FILES] = \ mpiops.run_once(update_ifg_list, ifg_files, params[C.INTERFEROGRAM_FILES]) if mpiops.rank == 0: with open(config.phase_closure_filtered_ifgs_list(params), 'w') as f: lines = [p.converted_path + '\n' for p in params[C.INTERFEROGRAM_FILES]] f.writelines(lines) # mask ifgs with nans where phase unwrap threshold is breached if mpiops.rank == 0: mask_pixels_with_unwrapping_errors(ifgs_breach_count, num_occurences_each_ifg, params) _create_ifg_dict(params) # update the preread_ifgs dict ifg_paths = [ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]] # update/save the phase_data in the tiled numpy files save_numpy_phase(ifg_paths, params) return params
def wrap_spatio_temporal_filter(params): """ A wrapper for the spatio-temporal filter so it can be tested. See docstring for spatio_temporal_filter. """ if params[cf.APSEST]: log.info('Doing APS spatio-temporal filtering') else: log.info('APS spatio-temporal filtering not required') return tiles = params[cf.TILES] preread_ifgs = params[cf.PREREAD_IFGS] ifg_paths = [ ifg_path.tmp_sampled_path for ifg_path in params[cf.INTERFEROGRAM_FILES] ] # perform some checks on existing ifgs log.debug('Checking APS correction status') if mpiops.run_once(shared.check_correction_status, ifg_paths, ifc.PYRATE_APS_ERROR): log.debug('Finished APS correction') return # return if True condition returned aps_error_files_on_disc = [ MultiplePaths.aps_error_path(i, params) for i in ifg_paths ] if all(a.exists() for a in aps_error_files_on_disc): log.warning("Reusing APS errors from previous run!!!") for ifg_path, a in mpiops.array_split( list(zip(ifg_paths, aps_error_files_on_disc))): phase = np.load(a) _save_aps_corrected_phase(ifg_path, phase) else: tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles) mpiops.comm.barrier() spatio_temporal_filter(tsincr, ifg_paths, params, preread_ifgs) mpiops.comm.barrier() shared.save_numpy_phase(ifg_paths, params)
def _create_ifg_dict(params): """ Save the preread_ifgs dict with information about the ifgs that are later used for fast loading of Ifg files in IfgPart class :param list dest_tifs: List of destination tifs :param dict params: Config dictionary :param list tiles: List of all Tile instances :return: preread_ifgs: Dictionary containing information regarding interferograms that are used later in workflow :rtype: dict """ dest_tifs = [ifg_path for ifg_path in params[C.INTERFEROGRAM_FILES]] ifgs_dict = {} process_tifs = mpiops.array_split(dest_tifs) for d in process_tifs: ifg = Ifg(d.tmp_sampled_path) # get the writable copy ifg.open() nan_and_mm_convert(ifg, params) ifgs_dict[d.tmp_sampled_path] = PrereadIfg( path=d.sampled_path, tmp_path=d.tmp_sampled_path, nan_fraction=ifg.nan_fraction, first=ifg.first, second=ifg.second, time_span=ifg.time_span, nrows=ifg.nrows, ncols=ifg.ncols, metadata=ifg.meta_data ) ifg.write_modified_phase() # update phase converted to mm ifg.close() ifgs_dict = join_dicts(mpiops.comm.allgather(ifgs_dict)) ifgs_dict = mpiops.run_once(__save_ifgs_dict_with_headers_and_epochs, dest_tifs, ifgs_dict, params, process_tifs) params[C.PREREAD_IFGS] = ifgs_dict return ifgs_dict
def _create_ifg_dict(params): """ 1. Convert ifg phase data into numpy binary files. 2. Save the preread_ifgs dict with information about the ifgs that are later used for fast loading of Ifg files in IfgPart class :param list dest_tifs: List of destination tifs :param dict params: Config dictionary :param list tiles: List of all Tile instances :return: preread_ifgs: Dictionary containing information regarding interferograms that are used later in workflow :rtype: dict """ dest_tifs = [ifg_path for ifg_path in params[cf.INTERFEROGRAM_FILES]] ifgs_dict = {} process_tifs = mpiops.array_split(dest_tifs) for d in process_tifs: ifg = shared._prep_ifg(d.sampled_path, params) ifgs_dict[d.tmp_sampled_path] = PrereadIfg( path=d.sampled_path, tmp_path=d.tmp_sampled_path, nan_fraction=ifg.nan_fraction, first=ifg.first, second=ifg.second, time_span=ifg.time_span, nrows=ifg.nrows, ncols=ifg.ncols, metadata=ifg.meta_data) ifg.close() ifgs_dict = join_dicts(mpiops.comm.allgather(ifgs_dict)) ifgs_dict = mpiops.run_once(__save_ifgs_dict_with_headers_and_epochs, dest_tifs, ifgs_dict, params, process_tifs) params[cf.PREREAD_IFGS] = ifgs_dict log.debug('Finished converting phase_data to numpy in process {}'.format( mpiops.rank)) return ifgs_dict