def _tlpfilter(nanmat, rows, cols, cutoff, span, threshold, tsincr, func): """ Wrapper function for temporal low pass filter """ tsfilt_incr_each_row = {} process_rows = mpiops.array_split(list(range(rows))) for r in process_rows: tsfilt_incr_each_row[r] = np.empty(tsincr.shape[1:], dtype=np.float32) * np.nan for j in range(cols): sel = np.nonzero(nanmat[r, j, :])[0] # don't select if nan m = len(sel) if m >= threshold: for k in range(m): yr = span[sel] - span[sel[k]] wgt = func(m, yr, cutoff) wgt /= np.sum(wgt) tsfilt_incr_each_row[r][j, sel[k]] = np.sum( tsincr[r, j, sel] * wgt) tsfilt_incr_combined = shared.join_dicts( mpiops.comm.allgather(tsfilt_incr_each_row)) tsfilt_incr = np.array([v[1] for v in tsfilt_incr_combined.items()]) return tsfilt_incr
def maxvar_vcm_calc_wrapper(params): """ MPI wrapper for maxvar and vcmt computation """ preread_ifgs = params[cf.PREREAD_IFGS] ifg_paths = [ifg_path.tmp_sampled_path for ifg_path in params[cf.INTERFEROGRAM_FILES]] log.info('Calculating the temporal variance-covariance matrix') def _get_r_dist(ifg_path): """ Get RDIst class object """ ifg = Ifg(ifg_path) ifg.open() r_dist = RDist(ifg)() ifg.close() return r_dist r_dist = mpiops.run_once(_get_r_dist, ifg_paths[0]) prcs_ifgs = mpiops.array_split(list(enumerate(ifg_paths))) process_maxvar = {} for n, i in prcs_ifgs: log.debug(f'Calculating maxvar for {n} of process ifgs {len(prcs_ifgs)} of total {len(ifg_paths)}') process_maxvar[int(n)] = cvd(i, params, r_dist, calc_alpha=True, write_vals=True, save_acg=True)[0] maxvar_d = shared.join_dicts(mpiops.comm.allgather(process_maxvar)) maxvar = [v[1] for v in sorted(maxvar_d.items(), key=lambda s: s[0])] vcmt = mpiops.run_once(get_vcmt, preread_ifgs, maxvar) log.debug("Finished maxvar and vcm calc!") params[cf.MAXVAR], params[cf.VCMT] = maxvar, vcmt np.save(Configuration.vcmt_path(params), arr=vcmt) return maxvar, vcmt
def spatial_low_pass_filter(ts_lp, ifg, params): """ Filter time series data spatially using either a Butterworth or Gaussian low pass filter defined by a cut-off distance. If the cut-off distance is defined as zero in the parameters dictionary then it is calculated for each time step using the pyrate.covariance.cvd_from_phase method. :param ndarray ts_lp: Array of time series data, the result of a temporal low pass filter operation. shape (ifg.shape, n_epochs) :param shared.Ifg instance ifg: interferogram object :param dict params: Dictionary of configuration parameters :return: ts_hp: filtered time series data of shape (ifg.shape, n_epochs) :rtype: ndarray """ log.info('Applying spatial low-pass filter') if params[cf.SLPF_NANFILL] == 0: ts_lp[np.isnan(ts_lp)] = 0 # need it here for cvd and fft else: # optionally interpolate, operation is inplace _interpolate_nans(ts_lp, params[cf.SLPF_NANFILL_METHOD]) r_dist = RDist(ifg)() nvels = ts_lp.shape[2] process_nvel = mpiops.array_split(range(nvels)) process_ts_lp = {} for i in process_nvel: process_ts_lp[i] = _slpfilter(ts_lp[:, :, i], ifg, r_dist, params) ts_lp_d = shared.join_dicts(mpiops.comm.allgather(process_ts_lp)) ts_lp = np.dstack([v[1] for v in sorted(ts_lp_d.items())]) log.debug('Finished applying spatial low pass filter') return ts_lp
def spatial_low_pass_filter(ts_hp: np.ndarray, ifg: Ifg, params: dict) -> np.ndarray: """ Filter time series data spatially using a Gaussian low-pass filter defined by a cut-off distance. If the cut-off distance is defined as zero in the parameters dictionary then it is calculated for each time step using the pyrate.covariance.cvd_from_phase method. :param ts_hp: Array of temporal high-pass time series data, shape (ifg.shape, n_epochs) :param ifg: pyrate.core.shared.Ifg Class object. :param params: Dictionary of PyRate configuration parameters. :return: ts_lp: Low-pass filtered time series data of shape (ifg.shape, n_epochs). """ log.info('Applying spatial low-pass filter') nvels = ts_hp.shape[2] cutoff = params[C.SLPF_CUTOFF] # nanfill = params[cf.SLPF_NANFILL] # fillmethod = params[cf.SLPF_NANFILL_METHOD] if cutoff == 0: r_dist = RDist(ifg)() # only needed for cvd_for_phase else: r_dist = None log.info(f'Gaussian spatial filter cutoff is {cutoff:.3f} km for all ' f'{nvels} time-series images') process_nvel = mpiops.array_split(range(nvels)) process_ts_lp = {} for i in process_nvel: process_ts_lp[i] = _slpfilter(ts_hp[:, :, i], ifg, r_dist, params) ts_lp_d = shared.join_dicts(mpiops.comm.allgather(process_ts_lp)) ts_lp = np.dstack([v[1] for v in sorted(ts_lp_d.items())]) log.debug('Finished applying spatial low pass filter') return ts_lp
def _assemble_tsincr(ifg_paths, params, preread_ifgs, tiles, nvels): """ Helper function to reconstruct time series images from tiles """ # pre-allocate dest 3D array shape = preread_ifgs[ifg_paths[0]].shape tsincr_p = {} process_nvels = mpiops.array_split(range(nvels)) for i in process_nvels: tsincr_p[i] = assemble_tiles(shape, params[cf.TMPDIR], tiles, out_type='tsincr_aps', index=i) tsincr_g = shared.join_dicts(mpiops.comm.allgather(tsincr_p)) return np.dstack([v[1] for v in sorted(tsincr_g.items())])
def temporal_high_pass_filter(tsincr: np.ndarray, epochlist: EpochList, params: dict) -> np.ndarray: """ Isolate high-frequency components of time series data by subtracting low-pass components obtained using a Gaussian filter defined by a cut-off time period (in days). :param tsincr: Array of incremental time series data of shape (ifg.shape, n_epochs). :param epochlist: A pyrate.core.shared.EpochList Class instance. :param params: Dictionary of PyRate configuration parameters. :return: ts_hp: Filtered high frequency time series data; shape (ifg.shape, nepochs). """ log.info('Applying temporal high-pass filter') threshold = params[C.TLPF_PTHR] cutoff_day = params[C.TLPF_CUTOFF] if cutoff_day < 1 or type(cutoff_day) != int: raise ValueError(f'tlpf_cutoff must be an integer greater than or ' f'equal to 1 day. Value provided = {cutoff_day}') # convert cutoff in days to years cutoff_yr = cutoff_day / ifc.DAYS_PER_YEAR log.info(f'Gaussian temporal filter cutoff is {cutoff_day} days ' f'({cutoff_yr:.4f} years)') intv = np.diff(epochlist.spans) # time interval for the neighboring epochs span = epochlist.spans[:tsincr.shape[2]] + intv / 2 # accumulated time rows, cols = tsincr.shape[:2] tsfilt_row = {} process_rows = mpiops.array_split(list(range(rows))) for r in process_rows: tsfilt_row[r] = np.empty(tsincr.shape[1:], dtype=np.float32) * np.nan for j in range(cols): # Result of gaussian filter is low frequency time series tsfilt_row[r][j, :] = gaussian_temporal_filter( tsincr[r, j, :], cutoff_yr, span, threshold) tsfilt_combined = shared.join_dicts(mpiops.comm.allgather(tsfilt_row)) tsfilt = np.array([v[1] for v in tsfilt_combined.items()]) log.debug("Finished applying temporal high-pass filter") # Return the high-pass time series by subtracting low-pass result from input return tsincr - tsfilt
def __create_ifg_edge_dict(ifg_files: List[str], params: dict) -> Dict[Edge, IndexedIfg]: """Returns a dictionary of indexed ifg 'edges'""" ifg_files.sort() ifgs = [Ifg(i) for i in ifg_files] def _func(ifg, index): ifg.open() ifg.nodata_value = params[C.NO_DATA_VALUE] ifg.convert_to_nans() ifg.convert_to_radians() idx_ifg = IndexedIfg(index, IfgPhase(ifg.phase_data)) return idx_ifg process_ifgs = mpiops.array_split(list(enumerate(ifgs))) ret_combined = {} for idx, _ifg in process_ifgs: ret_combined[Edge(_ifg.first, _ifg.second)] = _func(_ifg, idx) _ifg.close() ret_combined = join_dicts(mpiops.comm.allgather(ret_combined)) return ret_combined
def _create_ifg_dict(params): """ Save the preread_ifgs dict with information about the ifgs that are later used for fast loading of Ifg files in IfgPart class :param list dest_tifs: List of destination tifs :param dict params: Config dictionary :param list tiles: List of all Tile instances :return: preread_ifgs: Dictionary containing information regarding interferograms that are used later in workflow :rtype: dict """ dest_tifs = [ifg_path for ifg_path in params[C.INTERFEROGRAM_FILES]] ifgs_dict = {} process_tifs = mpiops.array_split(dest_tifs) for d in process_tifs: ifg = Ifg(d.tmp_sampled_path) # get the writable copy ifg.open() nan_and_mm_convert(ifg, params) ifgs_dict[d.tmp_sampled_path] = PrereadIfg( path=d.sampled_path, tmp_path=d.tmp_sampled_path, nan_fraction=ifg.nan_fraction, first=ifg.first, second=ifg.second, time_span=ifg.time_span, nrows=ifg.nrows, ncols=ifg.ncols, metadata=ifg.meta_data ) ifg.write_modified_phase() # update phase converted to mm ifg.close() ifgs_dict = join_dicts(mpiops.comm.allgather(ifgs_dict)) ifgs_dict = mpiops.run_once(__save_ifgs_dict_with_headers_and_epochs, dest_tifs, ifgs_dict, params, process_tifs) params[C.PREREAD_IFGS] = ifgs_dict return ifgs_dict
def _create_ifg_dict(params): """ 1. Convert ifg phase data into numpy binary files. 2. Save the preread_ifgs dict with information about the ifgs that are later used for fast loading of Ifg files in IfgPart class :param list dest_tifs: List of destination tifs :param dict params: Config dictionary :param list tiles: List of all Tile instances :return: preread_ifgs: Dictionary containing information regarding interferograms that are used later in workflow :rtype: dict """ dest_tifs = [ifg_path for ifg_path in params[cf.INTERFEROGRAM_FILES]] ifgs_dict = {} process_tifs = mpiops.array_split(dest_tifs) for d in process_tifs: ifg = shared._prep_ifg(d.sampled_path, params) ifgs_dict[d.tmp_sampled_path] = PrereadIfg( path=d.sampled_path, tmp_path=d.tmp_sampled_path, nan_fraction=ifg.nan_fraction, first=ifg.first, second=ifg.second, time_span=ifg.time_span, nrows=ifg.nrows, ncols=ifg.ncols, metadata=ifg.meta_data) ifg.close() ifgs_dict = join_dicts(mpiops.comm.allgather(ifgs_dict)) ifgs_dict = mpiops.run_once(__save_ifgs_dict_with_headers_and_epochs, dest_tifs, ifgs_dict, params, process_tifs) params[cf.PREREAD_IFGS] = ifgs_dict log.debug('Finished converting phase_data to numpy in process {}'.format( mpiops.rank)) return ifgs_dict