def _maxvar_vcm_calc(ifg_paths, params, preread_ifgs):
"""
MPI wrapper for maxvar and vcmt computation
"""
log.info('Calculating the temporal variance-covariance matrix')
process_indices = mpiops.array_split(range(len(ifg_paths)))
def _get_r_dist(ifg_path):
"""
Get RDIst class object
"""
ifg = Ifg(ifg_path)
ifg.open()
r_dist = vcm_module.RDist(ifg)()
ifg.close()
return r_dist
r_dist = mpiops.run_once(_get_r_dist, ifg_paths[0])
prcs_ifgs = mpiops.array_split(ifg_paths)
process_maxvar = []
for n, i in enumerate(prcs_ifgs):
log.debug(
'Calculating maxvar for {} of process ifgs {} of total {}'.format(
n + 1, len(prcs_ifgs), len(ifg_paths)))
process_maxvar.append(
vcm_module.cvd(i,
params,
r_dist,
calc_alpha=True,
write_vals=True,
save_acg=True)[0])
if mpiops.rank == MASTER_PROCESS:
maxvar = np.empty(len(ifg_paths), dtype=np.float64)
maxvar[process_indices] = process_maxvar
for i in range(1, mpiops.size): # pragma: no cover
rank_indices = mpiops.array_split(range(len(ifg_paths)), i)
this_process_ref_phs = np.empty(len(rank_indices),
dtype=np.float64)
mpiops.comm.Recv(this_process_ref_phs, source=i, tag=i)
maxvar[rank_indices] = this_process_ref_phs
else: # pragma: no cover
maxvar = np.empty(len(ifg_paths), dtype=np.float64)
mpiops.comm.Send(np.array(process_maxvar, dtype=np.float64),
dest=MASTER_PROCESS,
tag=mpiops.rank)
mpiops.comm.barrier()
maxvar = mpiops.comm.bcast(maxvar, root=0)
vcmt = mpiops.run_once(vcm_module.get_vcmt, preread_ifgs, maxvar)
log.debug("Finished maxvar and vcm calc!")
return maxvar, vcmt
def _tlpfilter(nanmat, rows, cols, cutoff, span, threshold, tsincr, func):
"""
Wrapper function for temporal low pass filter
"""
tsfilt_incr_each_row = {}
process_rows = mpiops.array_split(list(range(rows)))
for r in process_rows:
tsfilt_incr_each_row[r] = np.empty(tsincr.shape[1:],
dtype=np.float32) * np.nan
for j in range(cols):
sel = np.nonzero(nanmat[r, j, :])[0] # don't select if nan
m = len(sel)
if m >= threshold:
for k in range(m):
yr = span[sel] - span[sel[k]]
wgt = func(m, yr, cutoff)
wgt /= np.sum(wgt)
tsfilt_incr_each_row[r][j, sel[k]] = np.sum(
tsincr[r, j, sel] * wgt)
tsfilt_incr_combined = shared.join_dicts(
mpiops.comm.allgather(tsfilt_incr_each_row))
tsfilt_incr = np.array([v[1] for v in tsfilt_incr_combined.items()])
return tsfilt_incr
def _ref_pixel_calc(ifg_paths, params):
"""
Wrapper for reference pixel calculation
"""
refx = params[cf.REFX]
refy = params[cf.REFY]
ifg = Ifg(ifg_paths[0])
ifg.open(readonly=True)
if refx == -1 or refy == -1:
log.info('Searching for best reference pixel location')
half_patch_size, thresh, grid = refpixel.ref_pixel_setup(
ifg_paths, params)
process_grid = mpiops.array_split(grid)
refpixel.save_ref_pixel_blocks(process_grid, half_patch_size,
ifg_paths, params)
mean_sds = refpixel._ref_pixel_mpi(process_grid, half_patch_size,
ifg_paths, thresh, params)
mean_sds = mpiops.comm.gather(mean_sds, root=0)
if mpiops.rank == MASTER_PROCESS:
mean_sds = np.hstack(mean_sds)
refy, refx = mpiops.run_once(refpixel.find_min_mean, mean_sds, grid)
log.info('Selected reference pixel coordinate: ({}, {})'.format(
refx, refy))
else:
log.info('Reusing reference pixel from config file: ({}, {})'.format(
refx, refy))
ifg.close()
return refx, refy
def _merge_linrate(params: dict) -> None:
"""
Merge linear rate outputs
"""
shape, tiles, ifgs_dict = mpiops.run_once(__merge_setup, params)
log.info('Merging and writing Linear Rate product geotiffs')
# read and assemble tile outputs
out_types = [
'linear_' + x
for x in ['rate', 'rsquared', 'error', 'intercept', 'samples']
]
process_out_types = mpiops.array_split(out_types)
for p_out_type in process_out_types:
out = assemble_tiles(shape,
params[C.TMPDIR],
tiles,
out_type=p_out_type)
__save_merged_files(ifgs_dict,
params,
out,
p_out_type,
savenpy=params["savenpy"])
mpiops.comm.barrier()
def _calc_svd_time_series(ifg_paths: List[str], params: dict,
preread_ifgs: dict, tiles: List[Tile]) -> np.ndarray:
"""
Helper function to obtain time series for spatio-temporal filter
using SVD method
"""
# Is there other existing functions that can perform this same job?
log.info('Calculating incremental time series via SVD method for APS '
'correction')
# copy params temporarily
new_params = deepcopy(params)
new_params[C.TIME_SERIES_METHOD] = 2 # use SVD method
process_tiles = mpiops.array_split(tiles)
nvels = None
for t in process_tiles:
log.debug(f'Calculating time series for tile {t.index} during APS '
f'correction')
ifgp = [shared.IfgPart(p, t, preread_ifgs, params) for p in ifg_paths]
mst_tile = np.load(Configuration.mst_path(params, t.index))
tsincr = time_series(ifgp, new_params, vcmt=None, mst=mst_tile)[0]
np.save(file=os.path.join(params[C.TMPDIR],
f'tsincr_aps_{t.index}.npy'),
arr=tsincr)
nvels = tsincr.shape[2]
nvels = mpiops.comm.bcast(nvels, root=0)
mpiops.comm.barrier()
# need to assemble tsincr from all processes
tsincr_g = _assemble_tsincr(ifg_paths, params, preread_ifgs, tiles, nvels)
log.debug('Finished calculating time series for spatio-temporal filter')
return tsincr_g
def update_refpix_metadata(ifg_paths, refx, refy, transform, params):
"""
Function that adds metadata about the chosen reference pixel to each interferogram.
"""
pyrate_refpix_lon, pyrate_refpix_lat = mpiops.run_once(convert_pixel_value_to_geographic_coordinate, refx, refy, transform)
process_ifgs_paths = mpiops.array_split(ifg_paths)
for ifg_file in process_ifgs_paths:
log.debug("Updating metadata for: "+ifg_file)
ifg = Ifg(ifg_file)
log.debug("Open dataset")
ifg.open(readonly=True)
nan_and_mm_convert(ifg, params)
half_patch_size = params["refchipsize"] // 2
x, y = refx, refy
log.debug("Extract reference pixel windows")
data = ifg.phase_data[y - half_patch_size: y + half_patch_size + 1,
x - half_patch_size: x + half_patch_size + 1]
log.debug("Calculate standard deviation for reference window")
stddev_ref_area = np.nanstd(data)
log.debug("Calculate mean for reference window")
mean_ref_area = np.nanmean(data)
ifg.add_metadata(**{
ifc.PYRATE_REFPIX_X: str(refx),
ifc.PYRATE_REFPIX_Y: str(refy),
ifc.PYRATE_REFPIX_LAT: str(pyrate_refpix_lat),
ifc.PYRATE_REFPIX_LON: str(pyrate_refpix_lon),
ifc.PYRATE_MEAN_REF_AREA: str(mean_ref_area),
ifc.PYRATE_STDDEV_REF_AREA: str(stddev_ref_area)
})
ifg.write_modified_phase()
ifg.close()
def maxvar_vcm_calc_wrapper(params):
"""
MPI wrapper for maxvar and vcmt computation
"""
preread_ifgs = params[cf.PREREAD_IFGS]
ifg_paths = [ifg_path.tmp_sampled_path for ifg_path in params[cf.INTERFEROGRAM_FILES]]
log.info('Calculating the temporal variance-covariance matrix')
def _get_r_dist(ifg_path):
"""
Get RDIst class object
"""
ifg = Ifg(ifg_path)
ifg.open()
r_dist = RDist(ifg)()
ifg.close()
return r_dist
r_dist = mpiops.run_once(_get_r_dist, ifg_paths[0])
prcs_ifgs = mpiops.array_split(list(enumerate(ifg_paths)))
process_maxvar = {}
for n, i in prcs_ifgs:
log.debug(f'Calculating maxvar for {n} of process ifgs {len(prcs_ifgs)} of total {len(ifg_paths)}')
process_maxvar[int(n)] = cvd(i, params, r_dist, calc_alpha=True, write_vals=True, save_acg=True)[0]
maxvar_d = shared.join_dicts(mpiops.comm.allgather(process_maxvar))
maxvar = [v[1] for v in sorted(maxvar_d.items(), key=lambda s: s[0])]
vcmt = mpiops.run_once(get_vcmt, preread_ifgs, maxvar)
log.debug("Finished maxvar and vcm calc!")
params[cf.MAXVAR], params[cf.VCMT] = maxvar, vcmt
np.save(Configuration.vcmt_path(params), arr=vcmt)
return maxvar, vcmt
def _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles):
"""
Helper function to obtain time series for spatio-temporal filter
using SVD method
"""
# Is there other existing functions that can perform this same job?
log.info('Calculating time series via SVD method for ' 'APS correction')
# copy params temporarily
new_params = deepcopy(params)
new_params[cf.TIME_SERIES_METHOD] = 2 # use SVD method
process_tiles = mpiops.array_split(tiles)
nvels = None
for t in process_tiles:
log.debug('Calculating time series for tile {} during APS '
'correction'.format(t.index))
ifg_parts = [
shared.IfgPart(p, t, preread_ifgs, params) for p in ifg_paths
]
mst_tile = np.load(
os.path.join(params[cf.TMPDIR], 'mst_mat_{}.npy'.format(t.index)))
tsincr = time_series(ifg_parts, new_params, vcmt=None, mst=mst_tile)[0]
np.save(file=os.path.join(params[cf.TMPDIR],
'tsincr_aps_{}.npy'.format(t.index)),
arr=tsincr)
nvels = tsincr.shape[2]
nvels = mpiops.comm.bcast(nvels, root=0)
mpiops.comm.barrier()
# need to assemble tsincr from all processes
tsincr_g = mpiops.run_once(_assemble_tsincr, ifg_paths, params,
preread_ifgs, tiles, nvels)
log.debug('Finished calculating time series for spatio-temporal filter')
return tsincr_g
def save_numpy_phase(ifg_paths, tiles, params):
"""
Save interferogram phase data as numpy array file on disk.
:param list ifg_paths: List of strings for interferogram paths
:param list tiles: List of pyrate.shared.Tile instances
:param dict params: Dictionary of configuration parameters
:return: None, file saved to disk
"""
process_ifgs = mpiops.array_split(ifg_paths)
outdir = params[cf.TMPDIR]
if not os.path.exists(outdir):
mkdir_p(outdir)
for ifg_path in process_ifgs:
ifg = Ifg(ifg_path)
ifg.open()
phase_data = ifg.phase_data
bname = basename(ifg_path).split('.')[0]
for t in tiles:
p_data = phase_data[t.top_left_y:t.bottom_right_y,
t.top_left_x:t.bottom_right_x]
phase_file = 'phase_data_{}_{}.npy'.format(bname, t.index)
np.save(file=join(outdir, phase_file),
arr=p_data)
ifg.close()
mpiops.comm.barrier()
def spatial_low_pass_filter(ts_hp: np.ndarray, ifg: Ifg,
params: dict) -> np.ndarray:
"""
Filter time series data spatially using a Gaussian low-pass
filter defined by a cut-off distance. If the cut-off distance is
defined as zero in the parameters dictionary then it is calculated for
each time step using the pyrate.covariance.cvd_from_phase method.
:param ts_hp: Array of temporal high-pass time series data, shape (ifg.shape, n_epochs)
:param ifg: pyrate.core.shared.Ifg Class object.
:param params: Dictionary of PyRate configuration parameters.
:return: ts_lp: Low-pass filtered time series data of shape (ifg.shape, n_epochs).
"""
log.info('Applying spatial low-pass filter')
nvels = ts_hp.shape[2]
cutoff = params[C.SLPF_CUTOFF]
# nanfill = params[cf.SLPF_NANFILL]
# fillmethod = params[cf.SLPF_NANFILL_METHOD]
if cutoff == 0:
r_dist = RDist(ifg)() # only needed for cvd_for_phase
else:
r_dist = None
log.info(f'Gaussian spatial filter cutoff is {cutoff:.3f} km for all '
f'{nvels} time-series images')
process_nvel = mpiops.array_split(range(nvels))
process_ts_lp = {}
for i in process_nvel:
process_ts_lp[i] = _slpfilter(ts_hp[:, :, i], ifg, r_dist, params)
ts_lp_d = shared.join_dicts(mpiops.comm.allgather(process_ts_lp))
ts_lp = np.dstack([v[1] for v in sorted(ts_lp_d.items())])
log.debug('Finished applying spatial low pass filter')
return ts_lp
def _orb_fit_calc(ifg_paths, params, preread_ifgs=None):
"""
MPI wrapper for orbital fit correction
"""
log.info('Calculating orbital correction')
if not params[cf.ORBITAL_FIT]:
log.info('Orbital correction not required')
return
if preread_ifgs: # don't check except for mpi tests
# perform some general error/sanity checks
log.debug('Checking Orbital error correction status')
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_ORBITAL_ERROR):
log.debug('Finished Orbital error correction')
return # return if True condition returned
if params[cf.ORBITAL_FIT_METHOD] == 1:
prcs_ifgs = mpiops.array_split(ifg_paths)
orbital.remove_orbital_error(prcs_ifgs, params, preread_ifgs)
else:
# Here we do all the multilooking in one process, but in memory
# can use multiple processes if we write data to disc during
# remove_orbital_error step
# A performance comparison should be made for saving multilooked
# files on disc vs in memory single process multilooking
if mpiops.rank == MASTER_PROCESS:
orbital.remove_orbital_error(ifg_paths, params, preread_ifgs)
mpiops.comm.barrier()
log.debug('Finished Orbital error correction')
def _ts_to_ifgs(tsincr, preread_ifgs, params):
"""
Function that converts an incremental displacement time series into
interferometric phase observations. Used to re-construct an interferogram
network from a time series.
:param ndarray tsincr: incremental time series array of size
(ifg.shape, nepochs-1)
:param dict preread_ifgs: Dictionary of shared.PrereadIfg class instances
:return: None, interferograms are saved to disk
"""
log.debug('Reconstructing interferometric observations from time series')
ifgs = list(OrderedDict(sorted(preread_ifgs.items())).values())
_, n = mpiops.run_once(get_epochs, ifgs)
index_first, index_second = n[:len(ifgs)], n[len(ifgs):]
num_ifgs_tuples = mpiops.array_split(list(enumerate(ifgs)))
num_ifgs_tuples = [(int(num), ifg) for num, ifg in num_ifgs_tuples]
for i, ifg in num_ifgs_tuples:
aps_correction_on_disc = MultiplePaths.aps_error_path(
ifg.tmp_path, params)
phase = np.sum(tsincr[:, :, index_first[i]:index_second[i]], axis=2)
np.save(file=aps_correction_on_disc, arr=phase)
_save_aps_corrected_phase(ifg.tmp_path, phase)
def spatial_low_pass_filter(ts_lp, ifg, params):
"""
Filter time series data spatially using either a Butterworth or Gaussian
low pass filter defined by a cut-off distance. If the cut-off distance is
defined as zero in the parameters dictionary then it is calculated for
each time step using the pyrate.covariance.cvd_from_phase method.
:param ndarray ts_lp: Array of time series data, the result of a temporal
low pass filter operation. shape (ifg.shape, n_epochs)
:param shared.Ifg instance ifg: interferogram object
:param dict params: Dictionary of configuration parameters
:return: ts_hp: filtered time series data of shape (ifg.shape, n_epochs)
:rtype: ndarray
"""
log.info('Applying spatial low-pass filter')
if params[cf.SLPF_NANFILL] == 0:
ts_lp[np.isnan(ts_lp)] = 0 # need it here for cvd and fft
else:
# optionally interpolate, operation is inplace
_interpolate_nans(ts_lp, params[cf.SLPF_NANFILL_METHOD])
r_dist = RDist(ifg)()
nvels = ts_lp.shape[2]
process_nvel = mpiops.array_split(range(nvels))
process_ts_lp = {}
for i in process_nvel:
process_ts_lp[i] = _slpfilter(ts_lp[:, :, i], ifg, r_dist, params)
ts_lp_d = shared.join_dicts(mpiops.comm.allgather(process_ts_lp))
ts_lp = np.dstack([v[1] for v in sorted(ts_lp_d.items())])
log.debug('Finished applying spatial low pass filter')
return ts_lp
def _timeseries_calc(ifg_paths, params, vcmt, tiles, preread_ifgs):
"""
MPI wrapper for time series calculation.
"""
if params[cf.TIME_SERIES_CAL] == 0:
log.info('Time Series Calculation not required')
return
if params[cf.TIME_SERIES_METHOD] == 1:
log.info('Calculating time series using Laplacian Smoothing method')
elif params[cf.TIME_SERIES_METHOD] == 2:
log.info('Calculating time series using SVD method')
output_dir = params[cf.TMPDIR]
process_tiles = mpiops.array_split(tiles)
for t in process_tiles:
log.debug('Calculating time series for tile {}'.format(t.index))
ifg_parts = [shared.IfgPart(p, t, preread_ifgs) for p in ifg_paths]
mst_tile = np.load(
os.path.join(output_dir, 'mst_mat_{}.npy'.format(t.index)))
res = timeseries.time_series(ifg_parts, params, vcmt, mst_tile)
tsincr, tscum, _ = res
np.save(file=os.path.join(output_dir, 'tsincr_{}.npy'.format(t.index)),
arr=tsincr)
np.save(file=os.path.join(output_dir, 'tscuml_{}.npy'.format(t.index)),
arr=tscum)
mpiops.comm.barrier()
def iterable_split(func: Callable, iterable: Iterable, params: dict, *args,
**kwargs) -> np.ndarray:
"""
# TODO: a faster version using buffer-provider objects via the uppercase communication method
A faster version of iterable/tiles_split is possible when the return values from each process is of the same size
and will be addressed in future. In this case a buffer-provider object can be sent between processes using the
uppercase communication (like Gather instead of gather) methods which can be significantly faster.
"""
if params[C.PARALLEL]:
ret_combined = {}
rets = Parallel(n_jobs=params[C.PROCESSES],
verbose=joblib_log_level(C.LOG_LEVEL))(
delayed(func)(t, params, *args, **kwargs)
for t in iterable)
for i, r in enumerate(rets):
ret_combined[i] = r
else:
iterable_with_index = list(enumerate(iterable))
process_iterables = mpiops.array_split(iterable_with_index)
ret_combined = {}
for i, t in process_iterables:
ret_combined[i] = func(t, params, *args, **kwargs)
ret_combined = join_dicts(mpiops.comm.allgather(ret_combined))
ret = np.array([v[1] for v in ret_combined.items()], dtype=object)
mpiops.comm.barrier()
return ret
def save_numpy_phase(ifg_paths, params):
"""
Split interferogram phase data in to tiles (if they exist in the params
dict) and save as numpy array files on disk.
:param list ifg_paths: List of strings for interferogram paths
:param dict params: Dictionary of configuration parameters
:return: None, numpy file saved to disk
"""
tiles = params['tiles']
outdir = params[C.TMPDIR]
if not os.path.exists(outdir):
mkdir_p(outdir)
for ifg_path in mpiops.array_split(ifg_paths):
ifg = Ifg(ifg_path)
ifg.open()
phase_data = ifg.phase_data
bname = basename(ifg_path).split('.')[0]
for t in tiles:
p_data = phase_data[t.top_left_y:t.bottom_right_y,
t.top_left_x:t.bottom_right_x]
phase_file = 'phase_data_{}_{}.npy'.format(bname, t.index)
np.save(file=join(outdir, phase_file), arr=p_data)
ifg.close()
mpiops.comm.barrier()
log.debug(f'Finished writing phase_data to numpy files in {outdir}')
def _linrate_calc(ifg_paths, params, vcmt, tiles, preread_ifgs):
"""
MPI wrapper for linrate calculation
"""
process_tiles = mpiops.array_split(tiles)
log.info('Calculating rate map from stacking')
output_dir = params[cf.TMPDIR]
for t in process_tiles:
log.debug('Stacking of tile {}'.format(t.index))
ifg_parts = [shared.IfgPart(p, t, preread_ifgs) for p in ifg_paths]
mst_grid_n = np.load(
os.path.join(output_dir, 'mst_mat_{}.npy'.format(t.index)))
rate, error, samples = linrate.linear_rate(ifg_parts, params, vcmt,
mst_grid_n)
# declare file names
np.save(file=os.path.join(output_dir,
'linrate_{}.npy'.format(t.index)),
arr=rate)
np.save(file=os.path.join(output_dir,
'linerror_{}.npy'.format(t.index)),
arr=error)
np.save(file=os.path.join(output_dir,
'linsamples_{}.npy'.format(t.index)),
arr=samples)
mpiops.comm.barrier()
def _ref_phase_estimation(ifg_paths, params, refpx, refpy):
"""
Wrapper for reference phase estimation.
"""
log.info("Calculating reference phase and correcting each interferogram")
if len(ifg_paths) < 2:
raise rpe.ReferencePhaseError(
"At least two interferograms required for reference phase correction ({len_ifg_paths} "
"provided).".format(len_ifg_paths=len(ifg_paths)))
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_REF_PHASE):
log.debug('Finished reference phase correction')
return
if params[cf.REF_EST_METHOD] == 1:
ref_phs = rpe.est_ref_phase_method1(ifg_paths, params)
elif params[cf.REF_EST_METHOD] == 2:
ref_phs = rpe.est_ref_phase_method2(ifg_paths, params, refpx, refpy)
else:
raise rpe.ReferencePhaseError("No such option, use '1' or '2'.")
# Save reference phase numpy arrays to disk.
ref_phs_file = os.path.join(params[cf.TMPDIR], 'ref_phs.npy')
if mpiops.rank == MASTER_PROCESS:
collected_ref_phs = np.zeros(len(ifg_paths), dtype=np.float64)
process_indices = mpiops.array_split(range(len(ifg_paths)))
collected_ref_phs[process_indices] = ref_phs
for r in range(1, mpiops.size):
process_indices = mpiops.array_split(range(len(ifg_paths)), r)
this_process_ref_phs = np.zeros(shape=len(process_indices),
dtype=np.float64)
mpiops.comm.Recv(this_process_ref_phs, source=r, tag=r)
collected_ref_phs[process_indices] = this_process_ref_phs
np.save(file=ref_phs_file, arr=collected_ref_phs)
else:
mpiops.comm.Send(ref_phs, dest=MASTER_PROCESS, tag=mpiops.rank)
log.debug('Finished reference phase correction')
# Preserve old return value so tests don't break.
if isinstance(ifg_paths[0], Ifg):
ifgs = ifg_paths
else:
ifgs = [Ifg(ifg_path) for ifg_path in ifg_paths]
mpiops.comm.barrier()
return ref_phs, ifgs
def _copy_mlooked(params):
log.info(
"Copying input files into tempdir for manipulation during 'correct' steps"
)
mpaths = params[cf.INTERFEROGRAM_FILES]
process_mpaths = mpiops.array_split(mpaths)
for p in process_mpaths:
shutil.copy(p.sampled_path, p.tmp_sampled_path)
Path(p.tmp_sampled_path).chmod(
0o664) # assign write permission as prepifg output is readonly
def est_ref_phase_method2(ifg_paths, params, refpx, refpy):
"""
Reference phase estimation using method 2. Reference phase is the
median calculated with a patch around the supplied reference pixel.
:param list ifg_paths: List of interferogram paths or objects.
:param dict params: Dictionary of configuration parameters
:param int refpx: Reference pixel X found by ref pixel method
:param int refpy: Reference pixel Y found by ref pixel method
:return: ref_phs: Numpy array of reference phase values of size (nifgs, 1)
:rtype: ndarray
:return: ifgs: Reference phase data is removed interferograms in place
"""
half_chip_size = int(np.floor(params[cf.REF_CHIP_SIZE] / 2.0))
chipsize = 2 * half_chip_size + 1
thresh = chipsize * chipsize * params[cf.REF_MIN_FRAC]
def _inner(ifg_paths):
if isinstance(ifg_paths[0], Ifg):
ifgs = ifg_paths
else:
ifgs = [Ifg(ifg_path) for ifg_path in ifg_paths]
for ifg in ifgs:
if not ifg.is_open:
ifg.open(readonly=False)
phase_data = [i.phase_data for i in ifgs]
if params[cf.PARALLEL]:
ref_phs = Parallel(n_jobs=params[cf.PROCESSES],
verbose=joblib_log_level(cf.LOG_LEVEL))(
delayed(_est_ref_phs_method2)(
p, half_chip_size, refpx, refpy, thresh)
for p in phase_data)
for n, ifg in enumerate(ifgs):
ifg.phase_data -= ref_phs[n]
else:
ref_phs = np.zeros(len(ifgs))
for n, ifg in enumerate(ifgs):
ref_phs[n] = _est_ref_phs_method2(phase_data[n],
half_chip_size, refpx, refpy,
thresh)
ifg.phase_data -= ref_phs[n]
for ifg in ifgs:
_update_phase_metadata(ifg)
ifg.close()
return ref_phs
process_ifgs_paths = mpiops.array_split(ifg_paths)
ref_phs = _inner(process_ifgs_paths)
return ref_phs
def _update_phase_and_metadata(ifgs, ref_phs, params):
"""
Function that applies the reference phase correction and updates ifg metadata
"""
def __inner(ifg, ref_ph):
ifg.open()
# nan-convert and mm-convert before subtracting ref phase
nan_and_mm_convert(ifg, params)
# add 1e-20 to avoid 0.0 values being converted to NaN downstream (Github issue #310)
# TODO: implement a more robust way of avoiding this issue, e.g. using numpy masked
# arrays to mark invalid pixel values rather than directly changing values to NaN
ifg.phase_data -= ref_ph + 1e-20
ifg.meta_data[ifc.PYRATE_REF_PHASE] = ifc.REF_PHASE_REMOVED
ifg.write_modified_phase()
log.debug(f"Reference phase corrected for {ifg.data_path}")
ifg.close()
log.info("Correcting ifgs by subtracting reference phase")
for i, rp in zip(mpiops.array_split(ifgs), mpiops.array_split(ref_phs)):
__inner(i, rp)
def tiles_split(func, params, *args, **kwargs):
tiles = params[cf.TILES]
process_tiles = mpiops.array_split(tiles)
if params[cf.PARALLEL]:
Parallel(n_jobs=params[cf.PROCESSES],
verbose=joblib_log_level(cf.LOG_LEVEL))(
delayed(func)(t, params, *args, **kwargs)
for t in process_tiles)
else:
for t in process_tiles:
func(t, params, *args, **kwargs)
mpiops.comm.barrier()
def _merge_timeseries(rows, cols, params):
"""
Merge time series output
"""
log.info("Merging timeseries output")
shape, tiles, ifgs_dict = _merge_setup(rows, cols, params)
# load the first tsincr file to determine the number of time series tifs
tsincr_file = join(params[cf.TMPDIR], 'tsincr_0.npy')
tsincr = np.load(file=tsincr_file)
# pylint: disable=no-member
no_ts_tifs = tsincr.shape[2]
# create 2 x no_ts_tifs as we are splitting tsincr and tscuml to all processes.
process_tifs = mpiops.array_split(range(2 * no_ts_tifs))
# depending on nvelpar, this will not fit in memory
# e.g. nvelpar=100, nrows=10000, ncols=10000, 32bit floats need 40GB memory
# 32 * 100 * 10000 * 10000 / 8 bytes = 4e10 bytes = 40 GB
# the double for loop helps us overcome the memory limit
log.info('Process {} writing {} timeseries tifs of '
'total {}'.format(mpiops.rank, len(process_tifs), no_ts_tifs * 2))
for i in process_tifs:
if i < no_ts_tifs:
tscum_g = assemble_tiles(shape,
params[cf.TMPDIR],
tiles,
out_type='tscuml',
index=i)
_save_merged_files(ifgs_dict,
params[cf.OUT_DIR],
tscum_g,
out_type='tscuml',
index=i,
savenpy=params["savenpy"])
else:
i %= no_ts_tifs
tsincr_g = assemble_tiles(shape,
params[cf.TMPDIR],
tiles,
out_type='tsincr',
index=i)
_save_merged_files(ifgs_dict,
params[cf.OUT_DIR],
tsincr_g,
out_type='tsincr',
index=i,
savenpy=params["savenpy"])
mpiops.comm.barrier()
log.debug('Process {} finished writing {} timeseries tifs of '
'total {}'.format(mpiops.rank, len(process_tifs),
no_ts_tifs * 2))
def _create_ifg_dict(dest_tifs, params):
"""
1. Convert ifg phase data into numpy binary files.
2. Save the preread_ifgs dict with information about the ifgs that are
later used for fast loading of Ifg files in IfgPart class
:param list dest_tifs: List of destination tifs
:param dict params: Config dictionary
:param list tiles: List of all Tile instances
:return: preread_ifgs: Dictionary containing information regarding
interferograms that are used later in workflow
:rtype: dict
"""
ifgs_dict = {}
nifgs = len(dest_tifs)
process_tifs = mpiops.array_split(dest_tifs)
for d in process_tifs:
ifg = shared._prep_ifg(d, params)
ifgs_dict[d] = PrereadIfg(path=d,
nan_fraction=ifg.nan_fraction,
master=ifg.master,
slave=ifg.slave,
time_span=ifg.time_span,
nrows=ifg.nrows,
ncols=ifg.ncols,
metadata=ifg.meta_data)
ifg.close()
ifgs_dict = _join_dicts(mpiops.comm.allgather(ifgs_dict))
preread_ifgs_file = join(params[cf.TMPDIR], 'preread_ifgs.pk')
if mpiops.rank == MASTER_PROCESS:
# add some extra information that's also useful later
gt, md, wkt = shared.get_geotiff_header_info(process_tifs[0])
epochlist = algorithm.get_epochs(ifgs_dict)[0]
log.info(
'Found {} unique epochs in the {} interferogram network'.format(
len(epochlist.dates), nifgs))
ifgs_dict['epochlist'] = epochlist
ifgs_dict['gt'] = gt
ifgs_dict['md'] = md
ifgs_dict['wkt'] = wkt
# dump ifgs_dict file for later use
cp.dump(ifgs_dict, open(preread_ifgs_file, 'wb'))
mpiops.comm.barrier()
preread_ifgs = OrderedDict(
sorted(cp.load(open(preread_ifgs_file, 'rb')).items()))
log.debug('Finished converting phase_data to numpy in process {}'.format(
mpiops.rank))
return preread_ifgs
def est_ref_phase_method1(ifg_paths, params):
"""
Reference phase estimation using method 1. Reference phase is the
median of the whole interferogram image.
:param list ifg_paths: List of interferogram paths or objects
:param dict params: Dictionary of configuration parameters
:return: ref_phs: Numpy array of reference phase values of size (nifgs, 1)
:rtype: ndarray
:return: ifgs: Reference phase data is removed interferograms in place
"""
def _inner(ifg_paths):
if isinstance(ifg_paths[0], Ifg):
proc_ifgs = ifg_paths
else:
proc_ifgs = [Ifg(ifg_path) for ifg_path in ifg_paths]
for ifg in proc_ifgs:
if not ifg.is_open:
ifg.open(readonly=False)
ifg_phase_data_sum = np.zeros(proc_ifgs[0].shape, dtype=np.float64)
phase_data = [i.phase_data for i in proc_ifgs]
for ifg in proc_ifgs:
ifg_phase_data_sum += ifg.phase_data
comp = np.isnan(ifg_phase_data_sum)
comp = np.ravel(comp, order='F')
if params[cf.PARALLEL]:
log.info("Calculating ref phase using multiprocessing")
ref_phs = Parallel(n_jobs=params[cf.PROCESSES],
verbose=joblib_log_level(cf.LOG_LEVEL))(
delayed(_est_ref_phs_method1)(p, comp)
for p in phase_data)
for n, ifg in enumerate(proc_ifgs):
ifg.phase_data -= ref_phs[n]
else:
log.info("Calculating ref phase")
ref_phs = np.zeros(len(proc_ifgs))
for n, ifg in enumerate(proc_ifgs):
ref_phs[n] = _est_ref_phs_method1(ifg.phase_data, comp)
ifg.phase_data -= ref_phs[n]
for ifg in proc_ifgs:
_update_phase_metadata(ifg)
ifg.close()
return ref_phs
process_ifg_paths = mpiops.array_split(ifg_paths)
ref_phs = _inner(process_ifg_paths)
return ref_phs
def main(params: dict) -> None:
"""
PyRate merge main function. Assembles product tiles in to
single geotiff files
"""
if params[C.SIGNAL_POLARITY] == 1:
log.info(
f"Saving output products with the same sign convention as input data"
)
elif params[C.SIGNAL_POLARITY] == -1:
log.info(
f"Saving output products with reversed sign convention compared to the input data"
)
else:
log.warning(f"Check the value of signal_polarity parameter")
out_types = []
tsfile = join(params[C.TMPDIR], 'tscuml_0.npy')
if exists(tsfile):
_merge_timeseries(params, 'tscuml')
_merge_linrate(params)
out_types += ['linear_rate', 'linear_error', 'linear_rsquared']
# optional save of merged tsincr products
if params["savetsincr"] == 1:
_merge_timeseries(params, 'tsincr')
else:
log.warning(
'Not merging time series products; {} does not exist'.format(
tsfile))
stfile = join(params[C.TMPDIR], 'stack_rate_0.npy')
if exists(stfile):
# setup paths
mpiops.run_once(_merge_stack, params)
out_types += ['stack_rate', 'stack_error']
else:
log.warning(
'Not merging stack products; {} does not exist'.format(stfile))
if len(out_types) > 0:
process_out_types = mpiops.array_split(out_types)
for out_type in process_out_types:
create_png_and_kml_from_tif(params[C.VELOCITY_DIR],
output_type=out_type)
else:
log.warning('Exiting: no products to merge')
def _assemble_tsincr(ifg_paths, params, preread_ifgs, tiles, nvels):
"""
Helper function to reconstruct time series images from tiles
"""
# pre-allocate dest 3D array
shape = preread_ifgs[ifg_paths[0]].shape
tsincr_p = {}
process_nvels = mpiops.array_split(range(nvels))
for i in process_nvels:
tsincr_p[i] = assemble_tiles(shape,
params[cf.TMPDIR],
tiles,
out_type='tsincr_aps',
index=i)
tsincr_g = shared.join_dicts(mpiops.comm.allgather(tsincr_p))
return np.dstack([v[1] for v in sorted(tsincr_g.items())])
def temporal_high_pass_filter(tsincr: np.ndarray, epochlist: EpochList,
params: dict) -> np.ndarray:
"""
Isolate high-frequency components of time series data by subtracting
low-pass components obtained using a Gaussian filter defined by a
cut-off time period (in days).
:param tsincr: Array of incremental time series data of shape (ifg.shape, n_epochs).
:param epochlist: A pyrate.core.shared.EpochList Class instance.
:param params: Dictionary of PyRate configuration parameters.
:return: ts_hp: Filtered high frequency time series data; shape (ifg.shape, nepochs).
"""
log.info('Applying temporal high-pass filter')
threshold = params[C.TLPF_PTHR]
cutoff_day = params[C.TLPF_CUTOFF]
if cutoff_day < 1 or type(cutoff_day) != int:
raise ValueError(f'tlpf_cutoff must be an integer greater than or '
f'equal to 1 day. Value provided = {cutoff_day}')
# convert cutoff in days to years
cutoff_yr = cutoff_day / ifc.DAYS_PER_YEAR
log.info(f'Gaussian temporal filter cutoff is {cutoff_day} days '
f'({cutoff_yr:.4f} years)')
intv = np.diff(epochlist.spans) # time interval for the neighboring epochs
span = epochlist.spans[:tsincr.shape[2]] + intv / 2 # accumulated time
rows, cols = tsincr.shape[:2]
tsfilt_row = {}
process_rows = mpiops.array_split(list(range(rows)))
for r in process_rows:
tsfilt_row[r] = np.empty(tsincr.shape[1:], dtype=np.float32) * np.nan
for j in range(cols):
# Result of gaussian filter is low frequency time series
tsfilt_row[r][j, :] = gaussian_temporal_filter(
tsincr[r, j, :], cutoff_yr, span, threshold)
tsfilt_combined = shared.join_dicts(mpiops.comm.allgather(tsfilt_row))
tsfilt = np.array([v[1] for v in tsfilt_combined.items()])
log.debug("Finished applying temporal high-pass filter")
# Return the high-pass time series by subtracting low-pass result from input
return tsincr - tsfilt
def remove_orbital_error(ifgs: List, params: dict) -> None:
"""
Wrapper function for PyRate orbital error removal functionality.
NB: the ifg data is modified in situ, rather than create intermediate
files. The network method assumes the given ifgs have already been reduced
to a minimum spanning tree network.
"""
mpiops.run_once(__orb_params_check, params)
ifg_paths = [i.data_path
for i in ifgs] if isinstance(ifgs[0], Ifg) else ifgs
method = params[cf.ORBITAL_FIT_METHOD]
# mlooking is not necessary for independent correction in a computational sense
# can use multiple procesing if write_to_disc=True
if method == INDEPENDENT_METHOD:
log.info('Calculating orbital correction using independent method')
#TODO: implement multi-looking for independent orbit method
if params[cf.ORBITAL_FIT_LOOKS_X] > 1 or params[
cf.ORBITAL_FIT_LOOKS_Y] > 1:
log.warning(
'Multi-looking is not applied in independent orbit method')
ifgs = [shared.Ifg(p)
for p in ifg_paths] if isinstance(ifgs[0], str) else ifgs
process_ifgs = mpiops.array_split(ifgs)
for ifg in process_ifgs:
independent_orbital_correction(ifg, params=params)
elif method == NETWORK_METHOD:
log.info('Calculating orbital correction using network method')
# Here we do all the multilooking in one process, but in memory
# can use multiple processes if we write data to disc during
# remove_orbital_error step
# A performance comparison should be made for saving multilooked
# files on disc vs in memory single process multilooking
if mpiops.rank == MAIN_PROCESS:
mlooked = __create_multilooked_dataset_for_network_correction(
params)
_validate_mlooked(mlooked, ifg_paths)
network_orbital_correction(ifg_paths, params, mlooked)
else:
raise OrbitalError("Unrecognised orbital correction method")
def _mst_calc(dest_tifs, params, tiles, preread_ifgs):
"""
MPI wrapper function for MST calculation
"""
process_tiles = mpiops.array_split(tiles)
def _save_mst_tile(tile, i, preread_ifgs):
"""
Convenient inner loop for mst tile saving
"""
log.info('Calculating minimum spanning tree matrix')
mst_tile = mst.mst_multiprocessing(tile, dest_tifs, preread_ifgs)
# locally save the mst_mat
mst_file_process_n = join(params[cf.TMPDIR],
'mst_mat_{}.npy'.format(i))
np.save(file=mst_file_process_n, arr=mst_tile)
for t in process_tiles:
_save_mst_tile(t, t.index, preread_ifgs)
log.debug('Finished mst calculation for process {}'.format(mpiops.rank))
mpiops.comm.barrier()