def _tlpfilter(nanmat, rows, cols, cutoff, span, threshold, tsincr, func):
"""
Wrapper function for temporal low pass filter
"""
tsfilt_incr_each_row = {}
process_rows = mpiops.array_split(list(range(rows)))
for r in process_rows:
tsfilt_incr_each_row[r] = np.empty(tsincr.shape[1:],
dtype=np.float32) * np.nan
for j in range(cols):
sel = np.nonzero(nanmat[r, j, :])[0] # don't select if nan
m = len(sel)
if m >= threshold:
for k in range(m):
yr = span[sel] - span[sel[k]]
wgt = func(m, yr, cutoff)
wgt /= np.sum(wgt)
tsfilt_incr_each_row[r][j, sel[k]] = np.sum(
tsincr[r, j, sel] * wgt)
tsfilt_incr_combined = shared.join_dicts(
mpiops.comm.allgather(tsfilt_incr_each_row))
tsfilt_incr = np.array([v[1] for v in tsfilt_incr_combined.items()])
return tsfilt_incr
def maxvar_vcm_calc_wrapper(params):
"""
MPI wrapper for maxvar and vcmt computation
"""
preread_ifgs = params[cf.PREREAD_IFGS]
ifg_paths = [ifg_path.tmp_sampled_path for ifg_path in params[cf.INTERFEROGRAM_FILES]]
log.info('Calculating the temporal variance-covariance matrix')
def _get_r_dist(ifg_path):
"""
Get RDIst class object
"""
ifg = Ifg(ifg_path)
ifg.open()
r_dist = RDist(ifg)()
ifg.close()
return r_dist
r_dist = mpiops.run_once(_get_r_dist, ifg_paths[0])
prcs_ifgs = mpiops.array_split(list(enumerate(ifg_paths)))
process_maxvar = {}
for n, i in prcs_ifgs:
log.debug(f'Calculating maxvar for {n} of process ifgs {len(prcs_ifgs)} of total {len(ifg_paths)}')
process_maxvar[int(n)] = cvd(i, params, r_dist, calc_alpha=True, write_vals=True, save_acg=True)[0]
maxvar_d = shared.join_dicts(mpiops.comm.allgather(process_maxvar))
maxvar = [v[1] for v in sorted(maxvar_d.items(), key=lambda s: s[0])]
vcmt = mpiops.run_once(get_vcmt, preread_ifgs, maxvar)
log.debug("Finished maxvar and vcm calc!")
params[cf.MAXVAR], params[cf.VCMT] = maxvar, vcmt
np.save(Configuration.vcmt_path(params), arr=vcmt)
return maxvar, vcmt
def spatial_low_pass_filter(ts_lp, ifg, params):
"""
Filter time series data spatially using either a Butterworth or Gaussian
low pass filter defined by a cut-off distance. If the cut-off distance is
defined as zero in the parameters dictionary then it is calculated for
each time step using the pyrate.covariance.cvd_from_phase method.
:param ndarray ts_lp: Array of time series data, the result of a temporal
low pass filter operation. shape (ifg.shape, n_epochs)
:param shared.Ifg instance ifg: interferogram object
:param dict params: Dictionary of configuration parameters
:return: ts_hp: filtered time series data of shape (ifg.shape, n_epochs)
:rtype: ndarray
"""
log.info('Applying spatial low-pass filter')
if params[cf.SLPF_NANFILL] == 0:
ts_lp[np.isnan(ts_lp)] = 0 # need it here for cvd and fft
else:
# optionally interpolate, operation is inplace
_interpolate_nans(ts_lp, params[cf.SLPF_NANFILL_METHOD])
r_dist = RDist(ifg)()
nvels = ts_lp.shape[2]
process_nvel = mpiops.array_split(range(nvels))
process_ts_lp = {}
for i in process_nvel:
process_ts_lp[i] = _slpfilter(ts_lp[:, :, i], ifg, r_dist, params)
ts_lp_d = shared.join_dicts(mpiops.comm.allgather(process_ts_lp))
ts_lp = np.dstack([v[1] for v in sorted(ts_lp_d.items())])
log.debug('Finished applying spatial low pass filter')
return ts_lp
def spatial_low_pass_filter(ts_hp: np.ndarray, ifg: Ifg,
params: dict) -> np.ndarray:
"""
Filter time series data spatially using a Gaussian low-pass
filter defined by a cut-off distance. If the cut-off distance is
defined as zero in the parameters dictionary then it is calculated for
each time step using the pyrate.covariance.cvd_from_phase method.
:param ts_hp: Array of temporal high-pass time series data, shape (ifg.shape, n_epochs)
:param ifg: pyrate.core.shared.Ifg Class object.
:param params: Dictionary of PyRate configuration parameters.
:return: ts_lp: Low-pass filtered time series data of shape (ifg.shape, n_epochs).
"""
log.info('Applying spatial low-pass filter')
nvels = ts_hp.shape[2]
cutoff = params[C.SLPF_CUTOFF]
# nanfill = params[cf.SLPF_NANFILL]
# fillmethod = params[cf.SLPF_NANFILL_METHOD]
if cutoff == 0:
r_dist = RDist(ifg)() # only needed for cvd_for_phase
else:
r_dist = None
log.info(f'Gaussian spatial filter cutoff is {cutoff:.3f} km for all '
f'{nvels} time-series images')
process_nvel = mpiops.array_split(range(nvels))
process_ts_lp = {}
for i in process_nvel:
process_ts_lp[i] = _slpfilter(ts_hp[:, :, i], ifg, r_dist, params)
ts_lp_d = shared.join_dicts(mpiops.comm.allgather(process_ts_lp))
ts_lp = np.dstack([v[1] for v in sorted(ts_lp_d.items())])
log.debug('Finished applying spatial low pass filter')
return ts_lp
def _assemble_tsincr(ifg_paths, params, preread_ifgs, tiles, nvels):
"""
Helper function to reconstruct time series images from tiles
"""
# pre-allocate dest 3D array
shape = preread_ifgs[ifg_paths[0]].shape
tsincr_p = {}
process_nvels = mpiops.array_split(range(nvels))
for i in process_nvels:
tsincr_p[i] = assemble_tiles(shape,
params[cf.TMPDIR],
tiles,
out_type='tsincr_aps',
index=i)
tsincr_g = shared.join_dicts(mpiops.comm.allgather(tsincr_p))
return np.dstack([v[1] for v in sorted(tsincr_g.items())])
def temporal_high_pass_filter(tsincr: np.ndarray, epochlist: EpochList,
params: dict) -> np.ndarray:
"""
Isolate high-frequency components of time series data by subtracting
low-pass components obtained using a Gaussian filter defined by a
cut-off time period (in days).
:param tsincr: Array of incremental time series data of shape (ifg.shape, n_epochs).
:param epochlist: A pyrate.core.shared.EpochList Class instance.
:param params: Dictionary of PyRate configuration parameters.
:return: ts_hp: Filtered high frequency time series data; shape (ifg.shape, nepochs).
"""
log.info('Applying temporal high-pass filter')
threshold = params[C.TLPF_PTHR]
cutoff_day = params[C.TLPF_CUTOFF]
if cutoff_day < 1 or type(cutoff_day) != int:
raise ValueError(f'tlpf_cutoff must be an integer greater than or '
f'equal to 1 day. Value provided = {cutoff_day}')
# convert cutoff in days to years
cutoff_yr = cutoff_day / ifc.DAYS_PER_YEAR
log.info(f'Gaussian temporal filter cutoff is {cutoff_day} days '
f'({cutoff_yr:.4f} years)')
intv = np.diff(epochlist.spans) # time interval for the neighboring epochs
span = epochlist.spans[:tsincr.shape[2]] + intv / 2 # accumulated time
rows, cols = tsincr.shape[:2]
tsfilt_row = {}
process_rows = mpiops.array_split(list(range(rows)))
for r in process_rows:
tsfilt_row[r] = np.empty(tsincr.shape[1:], dtype=np.float32) * np.nan
for j in range(cols):
# Result of gaussian filter is low frequency time series
tsfilt_row[r][j, :] = gaussian_temporal_filter(
tsincr[r, j, :], cutoff_yr, span, threshold)
tsfilt_combined = shared.join_dicts(mpiops.comm.allgather(tsfilt_row))
tsfilt = np.array([v[1] for v in tsfilt_combined.items()])
log.debug("Finished applying temporal high-pass filter")
# Return the high-pass time series by subtracting low-pass result from input
return tsincr - tsfilt
def __create_ifg_edge_dict(ifg_files: List[str], params: dict) -> Dict[Edge, IndexedIfg]:
"""Returns a dictionary of indexed ifg 'edges'"""
ifg_files.sort()
ifgs = [Ifg(i) for i in ifg_files]
def _func(ifg, index):
ifg.open()
ifg.nodata_value = params[C.NO_DATA_VALUE]
ifg.convert_to_nans()
ifg.convert_to_radians()
idx_ifg = IndexedIfg(index, IfgPhase(ifg.phase_data))
return idx_ifg
process_ifgs = mpiops.array_split(list(enumerate(ifgs)))
ret_combined = {}
for idx, _ifg in process_ifgs:
ret_combined[Edge(_ifg.first, _ifg.second)] = _func(_ifg, idx)
_ifg.close()
ret_combined = join_dicts(mpiops.comm.allgather(ret_combined))
return ret_combined
def _create_ifg_dict(params):
"""
Save the preread_ifgs dict with information about the ifgs that are
later used for fast loading of Ifg files in IfgPart class
:param list dest_tifs: List of destination tifs
:param dict params: Config dictionary
:param list tiles: List of all Tile instances
:return: preread_ifgs: Dictionary containing information regarding
interferograms that are used later in workflow
:rtype: dict
"""
dest_tifs = [ifg_path for ifg_path in params[C.INTERFEROGRAM_FILES]]
ifgs_dict = {}
process_tifs = mpiops.array_split(dest_tifs)
for d in process_tifs:
ifg = Ifg(d.tmp_sampled_path) # get the writable copy
ifg.open()
nan_and_mm_convert(ifg, params)
ifgs_dict[d.tmp_sampled_path] = PrereadIfg(
path=d.sampled_path,
tmp_path=d.tmp_sampled_path,
nan_fraction=ifg.nan_fraction,
first=ifg.first,
second=ifg.second,
time_span=ifg.time_span,
nrows=ifg.nrows,
ncols=ifg.ncols,
metadata=ifg.meta_data
)
ifg.write_modified_phase() # update phase converted to mm
ifg.close()
ifgs_dict = join_dicts(mpiops.comm.allgather(ifgs_dict))
ifgs_dict = mpiops.run_once(__save_ifgs_dict_with_headers_and_epochs, dest_tifs, ifgs_dict, params, process_tifs)
params[C.PREREAD_IFGS] = ifgs_dict
return ifgs_dict
def _create_ifg_dict(params):
"""
1. Convert ifg phase data into numpy binary files.
2. Save the preread_ifgs dict with information about the ifgs that are
later used for fast loading of Ifg files in IfgPart class
:param list dest_tifs: List of destination tifs
:param dict params: Config dictionary
:param list tiles: List of all Tile instances
:return: preread_ifgs: Dictionary containing information regarding
interferograms that are used later in workflow
:rtype: dict
"""
dest_tifs = [ifg_path for ifg_path in params[cf.INTERFEROGRAM_FILES]]
ifgs_dict = {}
process_tifs = mpiops.array_split(dest_tifs)
for d in process_tifs:
ifg = shared._prep_ifg(d.sampled_path, params)
ifgs_dict[d.tmp_sampled_path] = PrereadIfg(
path=d.sampled_path,
tmp_path=d.tmp_sampled_path,
nan_fraction=ifg.nan_fraction,
first=ifg.first,
second=ifg.second,
time_span=ifg.time_span,
nrows=ifg.nrows,
ncols=ifg.ncols,
metadata=ifg.meta_data)
ifg.close()
ifgs_dict = join_dicts(mpiops.comm.allgather(ifgs_dict))
ifgs_dict = mpiops.run_once(__save_ifgs_dict_with_headers_and_epochs,
dest_tifs, ifgs_dict, params, process_tifs)
params[cf.PREREAD_IFGS] = ifgs_dict
log.debug('Finished converting phase_data to numpy in process {}'.format(
mpiops.rank))
return ifgs_dict