def _maxvar_vcm_calc(ifg_paths, params, preread_ifgs): """ MPI wrapper for maxvar and vcmt computation """ log.info('Calculating the temporal variance-covariance matrix') process_indices = mpiops.array_split(range(len(ifg_paths))) def _get_r_dist(ifg_path): """ Get RDIst class object """ ifg = Ifg(ifg_path) ifg.open() r_dist = vcm_module.RDist(ifg)() ifg.close() return r_dist r_dist = mpiops.run_once(_get_r_dist, ifg_paths[0]) prcs_ifgs = mpiops.array_split(ifg_paths) process_maxvar = [] for n, i in enumerate(prcs_ifgs): log.debug( 'Calculating maxvar for {} of process ifgs {} of total {}'.format( n + 1, len(prcs_ifgs), len(ifg_paths))) process_maxvar.append( vcm_module.cvd(i, params, r_dist, calc_alpha=True, write_vals=True, save_acg=True)[0]) if mpiops.rank == MASTER_PROCESS: maxvar = np.empty(len(ifg_paths), dtype=np.float64) maxvar[process_indices] = process_maxvar for i in range(1, mpiops.size): # pragma: no cover rank_indices = mpiops.array_split(range(len(ifg_paths)), i) this_process_ref_phs = np.empty(len(rank_indices), dtype=np.float64) mpiops.comm.Recv(this_process_ref_phs, source=i, tag=i) maxvar[rank_indices] = this_process_ref_phs else: # pragma: no cover maxvar = np.empty(len(ifg_paths), dtype=np.float64) mpiops.comm.Send(np.array(process_maxvar, dtype=np.float64), dest=MASTER_PROCESS, tag=mpiops.rank) mpiops.comm.barrier() maxvar = mpiops.comm.bcast(maxvar, root=0) vcmt = mpiops.run_once(vcm_module.get_vcmt, preread_ifgs, maxvar) log.debug("Finished maxvar and vcm calc!") return maxvar, vcmt
def _tlpfilter(nanmat, rows, cols, cutoff, span, threshold, tsincr, func): """ Wrapper function for temporal low pass filter """ tsfilt_incr_each_row = {} process_rows = mpiops.array_split(list(range(rows))) for r in process_rows: tsfilt_incr_each_row[r] = np.empty(tsincr.shape[1:], dtype=np.float32) * np.nan for j in range(cols): sel = np.nonzero(nanmat[r, j, :])[0] # don't select if nan m = len(sel) if m >= threshold: for k in range(m): yr = span[sel] - span[sel[k]] wgt = func(m, yr, cutoff) wgt /= np.sum(wgt) tsfilt_incr_each_row[r][j, sel[k]] = np.sum( tsincr[r, j, sel] * wgt) tsfilt_incr_combined = shared.join_dicts( mpiops.comm.allgather(tsfilt_incr_each_row)) tsfilt_incr = np.array([v[1] for v in tsfilt_incr_combined.items()]) return tsfilt_incr
def _ref_pixel_calc(ifg_paths, params): """ Wrapper for reference pixel calculation """ refx = params[cf.REFX] refy = params[cf.REFY] ifg = Ifg(ifg_paths[0]) ifg.open(readonly=True) if refx == -1 or refy == -1: log.info('Searching for best reference pixel location') half_patch_size, thresh, grid = refpixel.ref_pixel_setup( ifg_paths, params) process_grid = mpiops.array_split(grid) refpixel.save_ref_pixel_blocks(process_grid, half_patch_size, ifg_paths, params) mean_sds = refpixel._ref_pixel_mpi(process_grid, half_patch_size, ifg_paths, thresh, params) mean_sds = mpiops.comm.gather(mean_sds, root=0) if mpiops.rank == MASTER_PROCESS: mean_sds = np.hstack(mean_sds) refy, refx = mpiops.run_once(refpixel.find_min_mean, mean_sds, grid) log.info('Selected reference pixel coordinate: ({}, {})'.format( refx, refy)) else: log.info('Reusing reference pixel from config file: ({}, {})'.format( refx, refy)) ifg.close() return refx, refy
def _merge_linrate(params: dict) -> None: """ Merge linear rate outputs """ shape, tiles, ifgs_dict = mpiops.run_once(__merge_setup, params) log.info('Merging and writing Linear Rate product geotiffs') # read and assemble tile outputs out_types = [ 'linear_' + x for x in ['rate', 'rsquared', 'error', 'intercept', 'samples'] ] process_out_types = mpiops.array_split(out_types) for p_out_type in process_out_types: out = assemble_tiles(shape, params[C.TMPDIR], tiles, out_type=p_out_type) __save_merged_files(ifgs_dict, params, out, p_out_type, savenpy=params["savenpy"]) mpiops.comm.barrier()
def _calc_svd_time_series(ifg_paths: List[str], params: dict, preread_ifgs: dict, tiles: List[Tile]) -> np.ndarray: """ Helper function to obtain time series for spatio-temporal filter using SVD method """ # Is there other existing functions that can perform this same job? log.info('Calculating incremental time series via SVD method for APS ' 'correction') # copy params temporarily new_params = deepcopy(params) new_params[C.TIME_SERIES_METHOD] = 2 # use SVD method process_tiles = mpiops.array_split(tiles) nvels = None for t in process_tiles: log.debug(f'Calculating time series for tile {t.index} during APS ' f'correction') ifgp = [shared.IfgPart(p, t, preread_ifgs, params) for p in ifg_paths] mst_tile = np.load(Configuration.mst_path(params, t.index)) tsincr = time_series(ifgp, new_params, vcmt=None, mst=mst_tile)[0] np.save(file=os.path.join(params[C.TMPDIR], f'tsincr_aps_{t.index}.npy'), arr=tsincr) nvels = tsincr.shape[2] nvels = mpiops.comm.bcast(nvels, root=0) mpiops.comm.barrier() # need to assemble tsincr from all processes tsincr_g = _assemble_tsincr(ifg_paths, params, preread_ifgs, tiles, nvels) log.debug('Finished calculating time series for spatio-temporal filter') return tsincr_g
def update_refpix_metadata(ifg_paths, refx, refy, transform, params): """ Function that adds metadata about the chosen reference pixel to each interferogram. """ pyrate_refpix_lon, pyrate_refpix_lat = mpiops.run_once(convert_pixel_value_to_geographic_coordinate, refx, refy, transform) process_ifgs_paths = mpiops.array_split(ifg_paths) for ifg_file in process_ifgs_paths: log.debug("Updating metadata for: "+ifg_file) ifg = Ifg(ifg_file) log.debug("Open dataset") ifg.open(readonly=True) nan_and_mm_convert(ifg, params) half_patch_size = params["refchipsize"] // 2 x, y = refx, refy log.debug("Extract reference pixel windows") data = ifg.phase_data[y - half_patch_size: y + half_patch_size + 1, x - half_patch_size: x + half_patch_size + 1] log.debug("Calculate standard deviation for reference window") stddev_ref_area = np.nanstd(data) log.debug("Calculate mean for reference window") mean_ref_area = np.nanmean(data) ifg.add_metadata(**{ ifc.PYRATE_REFPIX_X: str(refx), ifc.PYRATE_REFPIX_Y: str(refy), ifc.PYRATE_REFPIX_LAT: str(pyrate_refpix_lat), ifc.PYRATE_REFPIX_LON: str(pyrate_refpix_lon), ifc.PYRATE_MEAN_REF_AREA: str(mean_ref_area), ifc.PYRATE_STDDEV_REF_AREA: str(stddev_ref_area) }) ifg.write_modified_phase() ifg.close()
def maxvar_vcm_calc_wrapper(params): """ MPI wrapper for maxvar and vcmt computation """ preread_ifgs = params[cf.PREREAD_IFGS] ifg_paths = [ifg_path.tmp_sampled_path for ifg_path in params[cf.INTERFEROGRAM_FILES]] log.info('Calculating the temporal variance-covariance matrix') def _get_r_dist(ifg_path): """ Get RDIst class object """ ifg = Ifg(ifg_path) ifg.open() r_dist = RDist(ifg)() ifg.close() return r_dist r_dist = mpiops.run_once(_get_r_dist, ifg_paths[0]) prcs_ifgs = mpiops.array_split(list(enumerate(ifg_paths))) process_maxvar = {} for n, i in prcs_ifgs: log.debug(f'Calculating maxvar for {n} of process ifgs {len(prcs_ifgs)} of total {len(ifg_paths)}') process_maxvar[int(n)] = cvd(i, params, r_dist, calc_alpha=True, write_vals=True, save_acg=True)[0] maxvar_d = shared.join_dicts(mpiops.comm.allgather(process_maxvar)) maxvar = [v[1] for v in sorted(maxvar_d.items(), key=lambda s: s[0])] vcmt = mpiops.run_once(get_vcmt, preread_ifgs, maxvar) log.debug("Finished maxvar and vcm calc!") params[cf.MAXVAR], params[cf.VCMT] = maxvar, vcmt np.save(Configuration.vcmt_path(params), arr=vcmt) return maxvar, vcmt
def _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles): """ Helper function to obtain time series for spatio-temporal filter using SVD method """ # Is there other existing functions that can perform this same job? log.info('Calculating time series via SVD method for ' 'APS correction') # copy params temporarily new_params = deepcopy(params) new_params[cf.TIME_SERIES_METHOD] = 2 # use SVD method process_tiles = mpiops.array_split(tiles) nvels = None for t in process_tiles: log.debug('Calculating time series for tile {} during APS ' 'correction'.format(t.index)) ifg_parts = [ shared.IfgPart(p, t, preread_ifgs, params) for p in ifg_paths ] mst_tile = np.load( os.path.join(params[cf.TMPDIR], 'mst_mat_{}.npy'.format(t.index))) tsincr = time_series(ifg_parts, new_params, vcmt=None, mst=mst_tile)[0] np.save(file=os.path.join(params[cf.TMPDIR], 'tsincr_aps_{}.npy'.format(t.index)), arr=tsincr) nvels = tsincr.shape[2] nvels = mpiops.comm.bcast(nvels, root=0) mpiops.comm.barrier() # need to assemble tsincr from all processes tsincr_g = mpiops.run_once(_assemble_tsincr, ifg_paths, params, preread_ifgs, tiles, nvels) log.debug('Finished calculating time series for spatio-temporal filter') return tsincr_g
def save_numpy_phase(ifg_paths, tiles, params): """ Save interferogram phase data as numpy array file on disk. :param list ifg_paths: List of strings for interferogram paths :param list tiles: List of pyrate.shared.Tile instances :param dict params: Dictionary of configuration parameters :return: None, file saved to disk """ process_ifgs = mpiops.array_split(ifg_paths) outdir = params[cf.TMPDIR] if not os.path.exists(outdir): mkdir_p(outdir) for ifg_path in process_ifgs: ifg = Ifg(ifg_path) ifg.open() phase_data = ifg.phase_data bname = basename(ifg_path).split('.')[0] for t in tiles: p_data = phase_data[t.top_left_y:t.bottom_right_y, t.top_left_x:t.bottom_right_x] phase_file = 'phase_data_{}_{}.npy'.format(bname, t.index) np.save(file=join(outdir, phase_file), arr=p_data) ifg.close() mpiops.comm.barrier()
def spatial_low_pass_filter(ts_hp: np.ndarray, ifg: Ifg, params: dict) -> np.ndarray: """ Filter time series data spatially using a Gaussian low-pass filter defined by a cut-off distance. If the cut-off distance is defined as zero in the parameters dictionary then it is calculated for each time step using the pyrate.covariance.cvd_from_phase method. :param ts_hp: Array of temporal high-pass time series data, shape (ifg.shape, n_epochs) :param ifg: pyrate.core.shared.Ifg Class object. :param params: Dictionary of PyRate configuration parameters. :return: ts_lp: Low-pass filtered time series data of shape (ifg.shape, n_epochs). """ log.info('Applying spatial low-pass filter') nvels = ts_hp.shape[2] cutoff = params[C.SLPF_CUTOFF] # nanfill = params[cf.SLPF_NANFILL] # fillmethod = params[cf.SLPF_NANFILL_METHOD] if cutoff == 0: r_dist = RDist(ifg)() # only needed for cvd_for_phase else: r_dist = None log.info(f'Gaussian spatial filter cutoff is {cutoff:.3f} km for all ' f'{nvels} time-series images') process_nvel = mpiops.array_split(range(nvels)) process_ts_lp = {} for i in process_nvel: process_ts_lp[i] = _slpfilter(ts_hp[:, :, i], ifg, r_dist, params) ts_lp_d = shared.join_dicts(mpiops.comm.allgather(process_ts_lp)) ts_lp = np.dstack([v[1] for v in sorted(ts_lp_d.items())]) log.debug('Finished applying spatial low pass filter') return ts_lp
def _orb_fit_calc(ifg_paths, params, preread_ifgs=None): """ MPI wrapper for orbital fit correction """ log.info('Calculating orbital correction') if not params[cf.ORBITAL_FIT]: log.info('Orbital correction not required') return if preread_ifgs: # don't check except for mpi tests # perform some general error/sanity checks log.debug('Checking Orbital error correction status') if mpiops.run_once(shared.check_correction_status, ifg_paths, ifc.PYRATE_ORBITAL_ERROR): log.debug('Finished Orbital error correction') return # return if True condition returned if params[cf.ORBITAL_FIT_METHOD] == 1: prcs_ifgs = mpiops.array_split(ifg_paths) orbital.remove_orbital_error(prcs_ifgs, params, preread_ifgs) else: # Here we do all the multilooking in one process, but in memory # can use multiple processes if we write data to disc during # remove_orbital_error step # A performance comparison should be made for saving multilooked # files on disc vs in memory single process multilooking if mpiops.rank == MASTER_PROCESS: orbital.remove_orbital_error(ifg_paths, params, preread_ifgs) mpiops.comm.barrier() log.debug('Finished Orbital error correction')
def _ts_to_ifgs(tsincr, preread_ifgs, params): """ Function that converts an incremental displacement time series into interferometric phase observations. Used to re-construct an interferogram network from a time series. :param ndarray tsincr: incremental time series array of size (ifg.shape, nepochs-1) :param dict preread_ifgs: Dictionary of shared.PrereadIfg class instances :return: None, interferograms are saved to disk """ log.debug('Reconstructing interferometric observations from time series') ifgs = list(OrderedDict(sorted(preread_ifgs.items())).values()) _, n = mpiops.run_once(get_epochs, ifgs) index_first, index_second = n[:len(ifgs)], n[len(ifgs):] num_ifgs_tuples = mpiops.array_split(list(enumerate(ifgs))) num_ifgs_tuples = [(int(num), ifg) for num, ifg in num_ifgs_tuples] for i, ifg in num_ifgs_tuples: aps_correction_on_disc = MultiplePaths.aps_error_path( ifg.tmp_path, params) phase = np.sum(tsincr[:, :, index_first[i]:index_second[i]], axis=2) np.save(file=aps_correction_on_disc, arr=phase) _save_aps_corrected_phase(ifg.tmp_path, phase)
def spatial_low_pass_filter(ts_lp, ifg, params): """ Filter time series data spatially using either a Butterworth or Gaussian low pass filter defined by a cut-off distance. If the cut-off distance is defined as zero in the parameters dictionary then it is calculated for each time step using the pyrate.covariance.cvd_from_phase method. :param ndarray ts_lp: Array of time series data, the result of a temporal low pass filter operation. shape (ifg.shape, n_epochs) :param shared.Ifg instance ifg: interferogram object :param dict params: Dictionary of configuration parameters :return: ts_hp: filtered time series data of shape (ifg.shape, n_epochs) :rtype: ndarray """ log.info('Applying spatial low-pass filter') if params[cf.SLPF_NANFILL] == 0: ts_lp[np.isnan(ts_lp)] = 0 # need it here for cvd and fft else: # optionally interpolate, operation is inplace _interpolate_nans(ts_lp, params[cf.SLPF_NANFILL_METHOD]) r_dist = RDist(ifg)() nvels = ts_lp.shape[2] process_nvel = mpiops.array_split(range(nvels)) process_ts_lp = {} for i in process_nvel: process_ts_lp[i] = _slpfilter(ts_lp[:, :, i], ifg, r_dist, params) ts_lp_d = shared.join_dicts(mpiops.comm.allgather(process_ts_lp)) ts_lp = np.dstack([v[1] for v in sorted(ts_lp_d.items())]) log.debug('Finished applying spatial low pass filter') return ts_lp
def _timeseries_calc(ifg_paths, params, vcmt, tiles, preread_ifgs): """ MPI wrapper for time series calculation. """ if params[cf.TIME_SERIES_CAL] == 0: log.info('Time Series Calculation not required') return if params[cf.TIME_SERIES_METHOD] == 1: log.info('Calculating time series using Laplacian Smoothing method') elif params[cf.TIME_SERIES_METHOD] == 2: log.info('Calculating time series using SVD method') output_dir = params[cf.TMPDIR] process_tiles = mpiops.array_split(tiles) for t in process_tiles: log.debug('Calculating time series for tile {}'.format(t.index)) ifg_parts = [shared.IfgPart(p, t, preread_ifgs) for p in ifg_paths] mst_tile = np.load( os.path.join(output_dir, 'mst_mat_{}.npy'.format(t.index))) res = timeseries.time_series(ifg_parts, params, vcmt, mst_tile) tsincr, tscum, _ = res np.save(file=os.path.join(output_dir, 'tsincr_{}.npy'.format(t.index)), arr=tsincr) np.save(file=os.path.join(output_dir, 'tscuml_{}.npy'.format(t.index)), arr=tscum) mpiops.comm.barrier()
def iterable_split(func: Callable, iterable: Iterable, params: dict, *args, **kwargs) -> np.ndarray: """ # TODO: a faster version using buffer-provider objects via the uppercase communication method A faster version of iterable/tiles_split is possible when the return values from each process is of the same size and will be addressed in future. In this case a buffer-provider object can be sent between processes using the uppercase communication (like Gather instead of gather) methods which can be significantly faster. """ if params[C.PARALLEL]: ret_combined = {} rets = Parallel(n_jobs=params[C.PROCESSES], verbose=joblib_log_level(C.LOG_LEVEL))( delayed(func)(t, params, *args, **kwargs) for t in iterable) for i, r in enumerate(rets): ret_combined[i] = r else: iterable_with_index = list(enumerate(iterable)) process_iterables = mpiops.array_split(iterable_with_index) ret_combined = {} for i, t in process_iterables: ret_combined[i] = func(t, params, *args, **kwargs) ret_combined = join_dicts(mpiops.comm.allgather(ret_combined)) ret = np.array([v[1] for v in ret_combined.items()], dtype=object) mpiops.comm.barrier() return ret
def save_numpy_phase(ifg_paths, params): """ Split interferogram phase data in to tiles (if they exist in the params dict) and save as numpy array files on disk. :param list ifg_paths: List of strings for interferogram paths :param dict params: Dictionary of configuration parameters :return: None, numpy file saved to disk """ tiles = params['tiles'] outdir = params[C.TMPDIR] if not os.path.exists(outdir): mkdir_p(outdir) for ifg_path in mpiops.array_split(ifg_paths): ifg = Ifg(ifg_path) ifg.open() phase_data = ifg.phase_data bname = basename(ifg_path).split('.')[0] for t in tiles: p_data = phase_data[t.top_left_y:t.bottom_right_y, t.top_left_x:t.bottom_right_x] phase_file = 'phase_data_{}_{}.npy'.format(bname, t.index) np.save(file=join(outdir, phase_file), arr=p_data) ifg.close() mpiops.comm.barrier() log.debug(f'Finished writing phase_data to numpy files in {outdir}')
def _linrate_calc(ifg_paths, params, vcmt, tiles, preread_ifgs): """ MPI wrapper for linrate calculation """ process_tiles = mpiops.array_split(tiles) log.info('Calculating rate map from stacking') output_dir = params[cf.TMPDIR] for t in process_tiles: log.debug('Stacking of tile {}'.format(t.index)) ifg_parts = [shared.IfgPart(p, t, preread_ifgs) for p in ifg_paths] mst_grid_n = np.load( os.path.join(output_dir, 'mst_mat_{}.npy'.format(t.index))) rate, error, samples = linrate.linear_rate(ifg_parts, params, vcmt, mst_grid_n) # declare file names np.save(file=os.path.join(output_dir, 'linrate_{}.npy'.format(t.index)), arr=rate) np.save(file=os.path.join(output_dir, 'linerror_{}.npy'.format(t.index)), arr=error) np.save(file=os.path.join(output_dir, 'linsamples_{}.npy'.format(t.index)), arr=samples) mpiops.comm.barrier()
def _ref_phase_estimation(ifg_paths, params, refpx, refpy): """ Wrapper for reference phase estimation. """ log.info("Calculating reference phase and correcting each interferogram") if len(ifg_paths) < 2: raise rpe.ReferencePhaseError( "At least two interferograms required for reference phase correction ({len_ifg_paths} " "provided).".format(len_ifg_paths=len(ifg_paths))) if mpiops.run_once(shared.check_correction_status, ifg_paths, ifc.PYRATE_REF_PHASE): log.debug('Finished reference phase correction') return if params[cf.REF_EST_METHOD] == 1: ref_phs = rpe.est_ref_phase_method1(ifg_paths, params) elif params[cf.REF_EST_METHOD] == 2: ref_phs = rpe.est_ref_phase_method2(ifg_paths, params, refpx, refpy) else: raise rpe.ReferencePhaseError("No such option, use '1' or '2'.") # Save reference phase numpy arrays to disk. ref_phs_file = os.path.join(params[cf.TMPDIR], 'ref_phs.npy') if mpiops.rank == MASTER_PROCESS: collected_ref_phs = np.zeros(len(ifg_paths), dtype=np.float64) process_indices = mpiops.array_split(range(len(ifg_paths))) collected_ref_phs[process_indices] = ref_phs for r in range(1, mpiops.size): process_indices = mpiops.array_split(range(len(ifg_paths)), r) this_process_ref_phs = np.zeros(shape=len(process_indices), dtype=np.float64) mpiops.comm.Recv(this_process_ref_phs, source=r, tag=r) collected_ref_phs[process_indices] = this_process_ref_phs np.save(file=ref_phs_file, arr=collected_ref_phs) else: mpiops.comm.Send(ref_phs, dest=MASTER_PROCESS, tag=mpiops.rank) log.debug('Finished reference phase correction') # Preserve old return value so tests don't break. if isinstance(ifg_paths[0], Ifg): ifgs = ifg_paths else: ifgs = [Ifg(ifg_path) for ifg_path in ifg_paths] mpiops.comm.barrier() return ref_phs, ifgs
def _copy_mlooked(params): log.info( "Copying input files into tempdir for manipulation during 'correct' steps" ) mpaths = params[cf.INTERFEROGRAM_FILES] process_mpaths = mpiops.array_split(mpaths) for p in process_mpaths: shutil.copy(p.sampled_path, p.tmp_sampled_path) Path(p.tmp_sampled_path).chmod( 0o664) # assign write permission as prepifg output is readonly
def est_ref_phase_method2(ifg_paths, params, refpx, refpy): """ Reference phase estimation using method 2. Reference phase is the median calculated with a patch around the supplied reference pixel. :param list ifg_paths: List of interferogram paths or objects. :param dict params: Dictionary of configuration parameters :param int refpx: Reference pixel X found by ref pixel method :param int refpy: Reference pixel Y found by ref pixel method :return: ref_phs: Numpy array of reference phase values of size (nifgs, 1) :rtype: ndarray :return: ifgs: Reference phase data is removed interferograms in place """ half_chip_size = int(np.floor(params[cf.REF_CHIP_SIZE] / 2.0)) chipsize = 2 * half_chip_size + 1 thresh = chipsize * chipsize * params[cf.REF_MIN_FRAC] def _inner(ifg_paths): if isinstance(ifg_paths[0], Ifg): ifgs = ifg_paths else: ifgs = [Ifg(ifg_path) for ifg_path in ifg_paths] for ifg in ifgs: if not ifg.is_open: ifg.open(readonly=False) phase_data = [i.phase_data for i in ifgs] if params[cf.PARALLEL]: ref_phs = Parallel(n_jobs=params[cf.PROCESSES], verbose=joblib_log_level(cf.LOG_LEVEL))( delayed(_est_ref_phs_method2)( p, half_chip_size, refpx, refpy, thresh) for p in phase_data) for n, ifg in enumerate(ifgs): ifg.phase_data -= ref_phs[n] else: ref_phs = np.zeros(len(ifgs)) for n, ifg in enumerate(ifgs): ref_phs[n] = _est_ref_phs_method2(phase_data[n], half_chip_size, refpx, refpy, thresh) ifg.phase_data -= ref_phs[n] for ifg in ifgs: _update_phase_metadata(ifg) ifg.close() return ref_phs process_ifgs_paths = mpiops.array_split(ifg_paths) ref_phs = _inner(process_ifgs_paths) return ref_phs
def _update_phase_and_metadata(ifgs, ref_phs, params): """ Function that applies the reference phase correction and updates ifg metadata """ def __inner(ifg, ref_ph): ifg.open() # nan-convert and mm-convert before subtracting ref phase nan_and_mm_convert(ifg, params) # add 1e-20 to avoid 0.0 values being converted to NaN downstream (Github issue #310) # TODO: implement a more robust way of avoiding this issue, e.g. using numpy masked # arrays to mark invalid pixel values rather than directly changing values to NaN ifg.phase_data -= ref_ph + 1e-20 ifg.meta_data[ifc.PYRATE_REF_PHASE] = ifc.REF_PHASE_REMOVED ifg.write_modified_phase() log.debug(f"Reference phase corrected for {ifg.data_path}") ifg.close() log.info("Correcting ifgs by subtracting reference phase") for i, rp in zip(mpiops.array_split(ifgs), mpiops.array_split(ref_phs)): __inner(i, rp)
def tiles_split(func, params, *args, **kwargs): tiles = params[cf.TILES] process_tiles = mpiops.array_split(tiles) if params[cf.PARALLEL]: Parallel(n_jobs=params[cf.PROCESSES], verbose=joblib_log_level(cf.LOG_LEVEL))( delayed(func)(t, params, *args, **kwargs) for t in process_tiles) else: for t in process_tiles: func(t, params, *args, **kwargs) mpiops.comm.barrier()
def _merge_timeseries(rows, cols, params): """ Merge time series output """ log.info("Merging timeseries output") shape, tiles, ifgs_dict = _merge_setup(rows, cols, params) # load the first tsincr file to determine the number of time series tifs tsincr_file = join(params[cf.TMPDIR], 'tsincr_0.npy') tsincr = np.load(file=tsincr_file) # pylint: disable=no-member no_ts_tifs = tsincr.shape[2] # create 2 x no_ts_tifs as we are splitting tsincr and tscuml to all processes. process_tifs = mpiops.array_split(range(2 * no_ts_tifs)) # depending on nvelpar, this will not fit in memory # e.g. nvelpar=100, nrows=10000, ncols=10000, 32bit floats need 40GB memory # 32 * 100 * 10000 * 10000 / 8 bytes = 4e10 bytes = 40 GB # the double for loop helps us overcome the memory limit log.info('Process {} writing {} timeseries tifs of ' 'total {}'.format(mpiops.rank, len(process_tifs), no_ts_tifs * 2)) for i in process_tifs: if i < no_ts_tifs: tscum_g = assemble_tiles(shape, params[cf.TMPDIR], tiles, out_type='tscuml', index=i) _save_merged_files(ifgs_dict, params[cf.OUT_DIR], tscum_g, out_type='tscuml', index=i, savenpy=params["savenpy"]) else: i %= no_ts_tifs tsincr_g = assemble_tiles(shape, params[cf.TMPDIR], tiles, out_type='tsincr', index=i) _save_merged_files(ifgs_dict, params[cf.OUT_DIR], tsincr_g, out_type='tsincr', index=i, savenpy=params["savenpy"]) mpiops.comm.barrier() log.debug('Process {} finished writing {} timeseries tifs of ' 'total {}'.format(mpiops.rank, len(process_tifs), no_ts_tifs * 2))
def _create_ifg_dict(dest_tifs, params): """ 1. Convert ifg phase data into numpy binary files. 2. Save the preread_ifgs dict with information about the ifgs that are later used for fast loading of Ifg files in IfgPart class :param list dest_tifs: List of destination tifs :param dict params: Config dictionary :param list tiles: List of all Tile instances :return: preread_ifgs: Dictionary containing information regarding interferograms that are used later in workflow :rtype: dict """ ifgs_dict = {} nifgs = len(dest_tifs) process_tifs = mpiops.array_split(dest_tifs) for d in process_tifs: ifg = shared._prep_ifg(d, params) ifgs_dict[d] = PrereadIfg(path=d, nan_fraction=ifg.nan_fraction, master=ifg.master, slave=ifg.slave, time_span=ifg.time_span, nrows=ifg.nrows, ncols=ifg.ncols, metadata=ifg.meta_data) ifg.close() ifgs_dict = _join_dicts(mpiops.comm.allgather(ifgs_dict)) preread_ifgs_file = join(params[cf.TMPDIR], 'preread_ifgs.pk') if mpiops.rank == MASTER_PROCESS: # add some extra information that's also useful later gt, md, wkt = shared.get_geotiff_header_info(process_tifs[0]) epochlist = algorithm.get_epochs(ifgs_dict)[0] log.info( 'Found {} unique epochs in the {} interferogram network'.format( len(epochlist.dates), nifgs)) ifgs_dict['epochlist'] = epochlist ifgs_dict['gt'] = gt ifgs_dict['md'] = md ifgs_dict['wkt'] = wkt # dump ifgs_dict file for later use cp.dump(ifgs_dict, open(preread_ifgs_file, 'wb')) mpiops.comm.barrier() preread_ifgs = OrderedDict( sorted(cp.load(open(preread_ifgs_file, 'rb')).items())) log.debug('Finished converting phase_data to numpy in process {}'.format( mpiops.rank)) return preread_ifgs
def est_ref_phase_method1(ifg_paths, params): """ Reference phase estimation using method 1. Reference phase is the median of the whole interferogram image. :param list ifg_paths: List of interferogram paths or objects :param dict params: Dictionary of configuration parameters :return: ref_phs: Numpy array of reference phase values of size (nifgs, 1) :rtype: ndarray :return: ifgs: Reference phase data is removed interferograms in place """ def _inner(ifg_paths): if isinstance(ifg_paths[0], Ifg): proc_ifgs = ifg_paths else: proc_ifgs = [Ifg(ifg_path) for ifg_path in ifg_paths] for ifg in proc_ifgs: if not ifg.is_open: ifg.open(readonly=False) ifg_phase_data_sum = np.zeros(proc_ifgs[0].shape, dtype=np.float64) phase_data = [i.phase_data for i in proc_ifgs] for ifg in proc_ifgs: ifg_phase_data_sum += ifg.phase_data comp = np.isnan(ifg_phase_data_sum) comp = np.ravel(comp, order='F') if params[cf.PARALLEL]: log.info("Calculating ref phase using multiprocessing") ref_phs = Parallel(n_jobs=params[cf.PROCESSES], verbose=joblib_log_level(cf.LOG_LEVEL))( delayed(_est_ref_phs_method1)(p, comp) for p in phase_data) for n, ifg in enumerate(proc_ifgs): ifg.phase_data -= ref_phs[n] else: log.info("Calculating ref phase") ref_phs = np.zeros(len(proc_ifgs)) for n, ifg in enumerate(proc_ifgs): ref_phs[n] = _est_ref_phs_method1(ifg.phase_data, comp) ifg.phase_data -= ref_phs[n] for ifg in proc_ifgs: _update_phase_metadata(ifg) ifg.close() return ref_phs process_ifg_paths = mpiops.array_split(ifg_paths) ref_phs = _inner(process_ifg_paths) return ref_phs
def main(params: dict) -> None: """ PyRate merge main function. Assembles product tiles in to single geotiff files """ if params[C.SIGNAL_POLARITY] == 1: log.info( f"Saving output products with the same sign convention as input data" ) elif params[C.SIGNAL_POLARITY] == -1: log.info( f"Saving output products with reversed sign convention compared to the input data" ) else: log.warning(f"Check the value of signal_polarity parameter") out_types = [] tsfile = join(params[C.TMPDIR], 'tscuml_0.npy') if exists(tsfile): _merge_timeseries(params, 'tscuml') _merge_linrate(params) out_types += ['linear_rate', 'linear_error', 'linear_rsquared'] # optional save of merged tsincr products if params["savetsincr"] == 1: _merge_timeseries(params, 'tsincr') else: log.warning( 'Not merging time series products; {} does not exist'.format( tsfile)) stfile = join(params[C.TMPDIR], 'stack_rate_0.npy') if exists(stfile): # setup paths mpiops.run_once(_merge_stack, params) out_types += ['stack_rate', 'stack_error'] else: log.warning( 'Not merging stack products; {} does not exist'.format(stfile)) if len(out_types) > 0: process_out_types = mpiops.array_split(out_types) for out_type in process_out_types: create_png_and_kml_from_tif(params[C.VELOCITY_DIR], output_type=out_type) else: log.warning('Exiting: no products to merge')
def _assemble_tsincr(ifg_paths, params, preread_ifgs, tiles, nvels): """ Helper function to reconstruct time series images from tiles """ # pre-allocate dest 3D array shape = preread_ifgs[ifg_paths[0]].shape tsincr_p = {} process_nvels = mpiops.array_split(range(nvels)) for i in process_nvels: tsincr_p[i] = assemble_tiles(shape, params[cf.TMPDIR], tiles, out_type='tsincr_aps', index=i) tsincr_g = shared.join_dicts(mpiops.comm.allgather(tsincr_p)) return np.dstack([v[1] for v in sorted(tsincr_g.items())])
def temporal_high_pass_filter(tsincr: np.ndarray, epochlist: EpochList, params: dict) -> np.ndarray: """ Isolate high-frequency components of time series data by subtracting low-pass components obtained using a Gaussian filter defined by a cut-off time period (in days). :param tsincr: Array of incremental time series data of shape (ifg.shape, n_epochs). :param epochlist: A pyrate.core.shared.EpochList Class instance. :param params: Dictionary of PyRate configuration parameters. :return: ts_hp: Filtered high frequency time series data; shape (ifg.shape, nepochs). """ log.info('Applying temporal high-pass filter') threshold = params[C.TLPF_PTHR] cutoff_day = params[C.TLPF_CUTOFF] if cutoff_day < 1 or type(cutoff_day) != int: raise ValueError(f'tlpf_cutoff must be an integer greater than or ' f'equal to 1 day. Value provided = {cutoff_day}') # convert cutoff in days to years cutoff_yr = cutoff_day / ifc.DAYS_PER_YEAR log.info(f'Gaussian temporal filter cutoff is {cutoff_day} days ' f'({cutoff_yr:.4f} years)') intv = np.diff(epochlist.spans) # time interval for the neighboring epochs span = epochlist.spans[:tsincr.shape[2]] + intv / 2 # accumulated time rows, cols = tsincr.shape[:2] tsfilt_row = {} process_rows = mpiops.array_split(list(range(rows))) for r in process_rows: tsfilt_row[r] = np.empty(tsincr.shape[1:], dtype=np.float32) * np.nan for j in range(cols): # Result of gaussian filter is low frequency time series tsfilt_row[r][j, :] = gaussian_temporal_filter( tsincr[r, j, :], cutoff_yr, span, threshold) tsfilt_combined = shared.join_dicts(mpiops.comm.allgather(tsfilt_row)) tsfilt = np.array([v[1] for v in tsfilt_combined.items()]) log.debug("Finished applying temporal high-pass filter") # Return the high-pass time series by subtracting low-pass result from input return tsincr - tsfilt
def remove_orbital_error(ifgs: List, params: dict) -> None: """ Wrapper function for PyRate orbital error removal functionality. NB: the ifg data is modified in situ, rather than create intermediate files. The network method assumes the given ifgs have already been reduced to a minimum spanning tree network. """ mpiops.run_once(__orb_params_check, params) ifg_paths = [i.data_path for i in ifgs] if isinstance(ifgs[0], Ifg) else ifgs method = params[cf.ORBITAL_FIT_METHOD] # mlooking is not necessary for independent correction in a computational sense # can use multiple procesing if write_to_disc=True if method == INDEPENDENT_METHOD: log.info('Calculating orbital correction using independent method') #TODO: implement multi-looking for independent orbit method if params[cf.ORBITAL_FIT_LOOKS_X] > 1 or params[ cf.ORBITAL_FIT_LOOKS_Y] > 1: log.warning( 'Multi-looking is not applied in independent orbit method') ifgs = [shared.Ifg(p) for p in ifg_paths] if isinstance(ifgs[0], str) else ifgs process_ifgs = mpiops.array_split(ifgs) for ifg in process_ifgs: independent_orbital_correction(ifg, params=params) elif method == NETWORK_METHOD: log.info('Calculating orbital correction using network method') # Here we do all the multilooking in one process, but in memory # can use multiple processes if we write data to disc during # remove_orbital_error step # A performance comparison should be made for saving multilooked # files on disc vs in memory single process multilooking if mpiops.rank == MAIN_PROCESS: mlooked = __create_multilooked_dataset_for_network_correction( params) _validate_mlooked(mlooked, ifg_paths) network_orbital_correction(ifg_paths, params, mlooked) else: raise OrbitalError("Unrecognised orbital correction method")
def _mst_calc(dest_tifs, params, tiles, preread_ifgs): """ MPI wrapper function for MST calculation """ process_tiles = mpiops.array_split(tiles) def _save_mst_tile(tile, i, preread_ifgs): """ Convenient inner loop for mst tile saving """ log.info('Calculating minimum spanning tree matrix') mst_tile = mst.mst_multiprocessing(tile, dest_tifs, preread_ifgs) # locally save the mst_mat mst_file_process_n = join(params[cf.TMPDIR], 'mst_mat_{}.npy'.format(i)) np.save(file=mst_file_process_n, arr=mst_tile) for t in process_tiles: _save_mst_tile(t, t.index, preread_ifgs) log.debug('Finished mst calculation for process {}'.format(mpiops.rank)) mpiops.comm.barrier()