def correct_ifgs(params: dict) -> None:
"""
Top level function to perform PyRate workflow on given interferograms
:param dict params: Dictionary of configuration parameters
:return: refpt: tuple of reference pixel x and y position
:rtype: tuple
:return: maxvar: array of maximum variance values of interferograms
:rtype: ndarray
:return: vcmt: Variance-covariance matrix array
:rtype: ndarray
"""
__validate_correct_steps(params)
# house keeping
_update_params_with_tiles(params)
_create_ifg_dict(params)
params[cf.REFX_FOUND], params[cf.REFY_FOUND] = ref_pixel_calc_wrapper(
params)
# run through the correct steps in user specified sequence
for step in params['correct']:
correct_steps[step](params)
log.info("Finished 'correct' step")
def _save_stack(ifgs_dict, params, tiles, out_type):
"""
Save stacking outputs
"""
log.info('Merging PyRate outputs {}'.format(out_type))
gt, md, wkt = ifgs_dict['gt'], ifgs_dict['md'], ifgs_dict['wkt']
epochlist = ifgs_dict['epochlist']
ifgs = [v for v in ifgs_dict.values() if isinstance(v, PrereadIfg)]
dest = os.path.join(params[cf.OUT_DIR], out_type + ".tif")
md[ifc.EPOCH_DATE] = epochlist.dates
if out_type == 'stack_rate':
md[ifc.DATA_TYPE] = ifc.STACKRATE
elif out_type == 'stack_error':
md[ifc.DATA_TYPE] = ifc.STACKERROR
else:
md[ifc.DATA_TYPE] = ifc.STACKSAMP
rate = np.zeros(shape=ifgs[0].shape, dtype=np.float32)
for t in tiles:
rate_file = os.path.join(params[cf.TMPDIR],
out_type + '_' + str(t.index) + '.npy')
rate_file = Path(rate_file)
rate_tile = np.load(file=rate_file)
rate[t.top_left_y:t.bottom_right_y,
t.top_left_x:t.bottom_right_x] = rate_tile
shared.write_output_geotiff(md, gt, wkt, rate, dest, np.nan)
npy_rate_file = os.path.join(params[cf.OUT_DIR], out_type + '.npy')
np.save(file=npy_rate_file, arr=rate)
log.debug('Finished PyRate merging {}'.format(out_type))
def __save_ifgs_dict_with_headers_and_epochs(dest_tifs, ifgs_dict, params,
process_tifs):
tmpdir = params[cf.TMPDIR]
if not os.path.exists(tmpdir):
shared.mkdir_p(tmpdir)
preread_ifgs_file = Configuration.preread_ifgs(params)
nifgs = len(dest_tifs)
# add some extra information that's also useful later
gt, md, wkt = shared.get_geotiff_header_info(
process_tifs[0].tmp_sampled_path)
epochlist = algorithm.get_epochs(ifgs_dict)[0]
log.info('Found {} unique epochs in the {} interferogram network'.format(
len(epochlist.dates), nifgs))
ifgs_dict['epochlist'] = epochlist
ifgs_dict['gt'] = gt
ifgs_dict['md'] = md
ifgs_dict['wkt'] = wkt
# dump ifgs_dict file for later use
cp.dump(ifgs_dict, open(preread_ifgs_file, 'wb'))
for k in ['gt', 'epochlist', 'md', 'wkt']:
ifgs_dict.pop(k)
return ifgs_dict
def _calc_svd_time_series(ifg_paths: List[str], params: dict,
preread_ifgs: dict, tiles: List[Tile]) -> np.ndarray:
"""
Helper function to obtain time series for spatio-temporal filter
using SVD method
"""
# Is there other existing functions that can perform this same job?
log.info('Calculating incremental time series via SVD method for APS '
'correction')
# copy params temporarily
new_params = deepcopy(params)
new_params[C.TIME_SERIES_METHOD] = 2 # use SVD method
process_tiles = mpiops.array_split(tiles)
nvels = None
for t in process_tiles:
log.debug(f'Calculating time series for tile {t.index} during APS '
f'correction')
ifgp = [shared.IfgPart(p, t, preread_ifgs, params) for p in ifg_paths]
mst_tile = np.load(Configuration.mst_path(params, t.index))
tsincr = time_series(ifgp, new_params, vcmt=None, mst=mst_tile)[0]
np.save(file=os.path.join(params[C.TMPDIR],
f'tsincr_aps_{t.index}.npy'),
arr=tsincr)
nvels = tsincr.shape[2]
nvels = mpiops.comm.bcast(nvels, root=0)
mpiops.comm.barrier()
# need to assemble tsincr from all processes
tsincr_g = _assemble_tsincr(ifg_paths, params, preread_ifgs, tiles, nvels)
log.debug('Finished calculating time series for spatio-temporal filter')
return tsincr_g
def _stack_calc(ifg_paths, params, vcmt, tiles, preread_ifgs):
"""
MPI wrapper for stacking calculation
"""
process_tiles = mpiops.array_split(tiles)
log.info('Calculating rate map from stacking')
output_dir = params[cf.TMPDIR]
for t in process_tiles:
log.info('Stacking of tile {}'.format(t.index))
ifg_parts = [
shared.IfgPart(p, t, preread_ifgs, params) for p in ifg_paths
]
mst_grid_n = np.load(
os.path.join(output_dir, 'mst_mat_{}.npy'.format(t.index)))
rate, error, samples = stack.stack_rate_array(ifg_parts, params, vcmt,
mst_grid_n)
# declare file names
np.save(file=os.path.join(output_dir,
'stack_rate_{}.npy'.format(t.index)),
arr=rate)
np.save(file=os.path.join(output_dir,
'stack_error_{}.npy'.format(t.index)),
arr=error)
np.save(file=os.path.join(output_dir,
'stack_samples_{}.npy'.format(t.index)),
arr=samples)
mpiops.comm.barrier()
log.debug("Finished stack rate calc!")
def main(params):
"""
Main workflow function for preparing interferograms for PyRate.
:param dict params: Parameters dictionary read in from the config file
"""
# TODO: looks like ifg_paths are ordered according to ifg list
# This probably won't be a problem because input list won't be reordered
# and the original gamma generated list is ordered) this may not affect
# the important pyrate stuff anyway, but might affect gen_thumbs.py.
# Going to assume ifg_paths is ordered correcly
# pylint: disable=too-many-branches
shared.mpi_vs_multiprocess_logging("prepifg", params)
ifg_paths = params[cf.INTERFEROGRAM_FILES]
if params[cf.DEM_FILE] is not None: # optional DEM conversion
ifg_paths.append(params[cf.DEM_FILE_PATH])
shared.mkdir_p(params[cf.OUT_DIR]) # create output dir
process_ifgs_paths = np.array_split(ifg_paths, mpiops.size)[mpiops.rank]
gtiff_paths = [p.converted_path for p in process_ifgs_paths]
do_prepifg(gtiff_paths, params)
mpiops.comm.barrier()
log.info("Finished prepifg")
def spatial_low_pass_filter(ts_hp: np.ndarray, ifg: Ifg,
params: dict) -> np.ndarray:
"""
Filter time series data spatially using a Gaussian low-pass
filter defined by a cut-off distance. If the cut-off distance is
defined as zero in the parameters dictionary then it is calculated for
each time step using the pyrate.covariance.cvd_from_phase method.
:param ts_hp: Array of temporal high-pass time series data, shape (ifg.shape, n_epochs)
:param ifg: pyrate.core.shared.Ifg Class object.
:param params: Dictionary of PyRate configuration parameters.
:return: ts_lp: Low-pass filtered time series data of shape (ifg.shape, n_epochs).
"""
log.info('Applying spatial low-pass filter')
nvels = ts_hp.shape[2]
cutoff = params[C.SLPF_CUTOFF]
# nanfill = params[cf.SLPF_NANFILL]
# fillmethod = params[cf.SLPF_NANFILL_METHOD]
if cutoff == 0:
r_dist = RDist(ifg)() # only needed for cvd_for_phase
else:
r_dist = None
log.info(f'Gaussian spatial filter cutoff is {cutoff:.3f} km for all '
f'{nvels} time-series images')
process_nvel = mpiops.array_split(range(nvels))
process_ts_lp = {}
for i in process_nvel:
process_ts_lp[i] = _slpfilter(ts_hp[:, :, i], ifg, r_dist, params)
ts_lp_d = shared.join_dicts(mpiops.comm.allgather(process_ts_lp))
ts_lp = np.dstack([v[1] for v in sorted(ts_lp_d.items())])
log.debug('Finished applying spatial low pass filter')
return ts_lp
def main(params):
"""
Main workflow function for preparing interferograms for PyRate.
:param dict params: Parameters dictionary read in from the config file
"""
# TODO: looks like ifg_paths are ordered according to ifg list
# This probably won't be a problem because input list won't be reordered
# and the original gamma generated list is ordered) this may not affect
# the important pyrate stuff anyway, but might affect gen_thumbs.py.
# Going to assume ifg_paths is ordered correcly
# pylint: disable=too-many-branches
shared.mpi_vs_multiprocess_logging("prepifg", params)
ifg_paths = params[cf.INTERFEROGRAM_FILES]
if params[cf.DEM_FILE] is not None: # optional DEM conversion
ifg_paths.append(params[cf.DEM_FILE_PATH])
if params[cf.COH_MASK]:
ifg_paths.extend(params[cf.COHERENCE_FILE_PATHS])
shared.mkdir_p(params[cf.OUT_DIR]) # create output dir
user_exts = (params[cf.IFG_XFIRST], params[cf.IFG_YFIRST], params[cf.IFG_XLAST], params[cf.IFG_YLAST])
xlooks, ylooks, crop = cf.transform_params(params)
ifgs = [prepifg_helper.dem_or_ifg(p.converted_path) for p in ifg_paths]
exts = prepifg_helper.get_analysis_extent(crop, ifgs, xlooks, ylooks, user_exts=user_exts)
process_ifgs_paths = np.array_split(ifg_paths, mpiops.size)[mpiops.rank]
do_prepifg(process_ifgs_paths, exts, params)
mpiops.comm.barrier()
log.info("Finished prepifg")
def coherence_masking(input_gdal_dataset: Dataset, coherence_file_path: str,
coherence_thresh: float) -> None:
"""
Perform coherence masking on raster in-place.
Based on gdal_calc formula provided by Nahidul:
gdal_calc.py -A 20151127-20151209_VV_8rlks_flat_eqa.cc.tif
-B 20151127-20151209_VV_8rlks_eqa.unw.tif
--outfile=test_v1.tif --calc="B*(A>=0.8)-999*(A<0.8)"
--NoDataValue=-999
"""
coherence_ds = gdal.Open(coherence_file_path, gdalconst.GA_ReadOnly)
coherence_band = coherence_ds.GetRasterBand(1)
src_band = input_gdal_dataset.GetRasterBand(1)
ndv = np.nan
coherence = coherence_band.ReadAsArray()
src = src_band.ReadAsArray()
var = {"coh": coherence, "src": src, "t": coherence_thresh, "ndv": ndv}
formula = "where(coh>=t, src, ndv)"
res = ne.evaluate(formula, local_dict=var)
src_band.WriteArray(res)
# update metadata
input_gdal_dataset.GetRasterBand(1).SetNoDataValue(ndv)
input_gdal_dataset.FlushCache() # write on the disc
log.info(f"Applied coherence masking using coh file {coherence_file_path}")
def mst_calc_wrapper(params):
"""
MPI wrapper function for MST calculation
"""
log.info('Calculating minimum spanning tree matrix')
def _save_mst_tile(tile: Tile, params: dict) -> None:
"""
Convenient inner loop for mst tile saving
"""
preread_ifgs = params[C.PREREAD_IFGS]
dest_tifs = [
ifg_path.tmp_sampled_path
for ifg_path in params[C.INTERFEROGRAM_FILES]
]
mst_file_process_n = Configuration.mst_path(params, index=tile.index)
if mst_file_process_n.exists():
return
mst_tile = mst_multiprocessing(tile, dest_tifs, preread_ifgs, params)
# locally save the mst_mat
np.save(file=mst_file_process_n, arr=mst_tile)
tiles_split(_save_mst_tile, params)
log.debug('Finished minimum spanning tree calculation')
def mask_rate(rate, error, maxsig):
"""
Function to mask pixels in the rate and error arrays when the error
is greater than the error threshold 'maxsig'.
:param ndarray rate: array of pixel rates derived by stacking
:param ndarray error: array of errors for the pixel rates
:param int maxsig: error threshold for masking (in millimetres).
:return: rate: Masked rate (velocity) map
:rtype: ndarray
:return: error: Masked error (standard deviation) map
:rtype: ndarray
"""
# initialise mask array with existing NaNs
mask = ~isnan(error)
# original Nan count
orig = np.count_nonzero(mask)
# determine where error is larger than the maximum sigma threshold
mask[mask] &= error[mask] > maxsig
# replace values with NaNs
rate[mask] = nan
error[mask] = nan
# calculate percentage of masked pixels
nummasked = int(np.count_nonzero(mask) / orig * 100)
log.info('Percentage of pixels masked = {}%'.format(nummasked))
return rate, error
def wrap_spatio_temporal_filter(ifg_paths, params, tiles, preread_ifgs):
"""
A wrapper for the spatio-temporal filter so it can be tested.
See docstring for spatio_temporal_filter.
"""
if params[cf.APSEST]:
log.info('Doing APS spatio-temporal filtering')
else:
log.info('APS spatio-temporal filtering not required')
return
# perform some checks on existing ifgs
log.debug('Checking APS correction status')
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_APS_ERROR):
log.debug('Finished APS correction')
return # return if True condition returned
tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles)
mpiops.comm.barrier()
ifg = Ifg(ifg_paths[0]) # just grab any for parameters in slpfilter
ifg.open()
spatio_temporal_filter(tsincr, ifg, params, preread_ifgs)
ifg.close()
mpiops.comm.barrier()
def _geotiff_multiprocessing(unw_path: MultiplePaths,
params: dict) -> Tuple[str, bool]:
"""
Multiprocessing wrapper for full-res geotiff conversion
"""
# TODO: Need a more robust method for identifying coherence files.
dest = unw_path.converted_path
processor = params[C.PROCESSOR] # roipac or gamma
# Create full-res geotiff if not already on disk
if not os.path.exists(dest):
if processor == GAMMA:
header = gamma.gamma_header(unw_path.unwrapped_path, params)
elif processor == ROIPAC:
log.info(
"Warning: ROI_PAC support will be deprecated in a future PyRate release"
)
header = roipac.roipac_header(unw_path.unwrapped_path, params)
else:
raise PreprocessError('Processor must be ROI_PAC (0) or GAMMA (1)')
header[ifc.INPUT_TYPE] = unw_path.input_type
shared.write_fullres_geotiff(header,
unw_path.unwrapped_path,
dest,
nodata=params[C.NO_DATA_VALUE])
Path(dest).chmod(0o444) # readonly output
return dest, True
else:
log.warning(
f"Full-res geotiff already exists in {dest}! Returning existing geotiff!"
)
return dest, False
def _merge_linrate(params: dict) -> None:
"""
Merge linear rate outputs
"""
shape, tiles, ifgs_dict = mpiops.run_once(__merge_setup, params)
log.info('Merging and writing Linear Rate product geotiffs')
# read and assemble tile outputs
out_types = [
'linear_' + x
for x in ['rate', 'rsquared', 'error', 'intercept', 'samples']
]
process_out_types = mpiops.array_split(out_types)
for p_out_type in process_out_types:
out = assemble_tiles(shape,
params[C.TMPDIR],
tiles,
out_type=p_out_type)
__save_merged_files(ifgs_dict,
params,
out,
p_out_type,
savenpy=params["savenpy"])
mpiops.comm.barrier()
def __drop_ifgs_if_not_part_of_any_loop(ifg_files: List[str],
loops: List[WeightedLoop],
params: dict) -> List[str]:
"""
Check if an ifg is part of any of the loops, otherwise drop it from the list of interferograms for further PyRate
processing.
"""
loop_ifgs = set()
for weighted_loop in loops:
for edge in weighted_loop.loop:
loop_ifgs.add(Edge(edge.first, edge.second))
ifgs = [Ifg(i) for i in ifg_files]
for i in ifgs:
i.open()
i.nodata_value = params[C.NO_DATA_VALUE]
selected_ifg_files = []
for i, f in zip(ifgs, ifg_files):
if Edge(i.first, i.second) in loop_ifgs:
selected_ifg_files.append(f)
if len(ifg_files) != len(selected_ifg_files):
log.info(
f'Only {len(selected_ifg_files)} (out of {len(ifg_files)}) ifgs participate in '
f'one or more closure loops, and are selected for further PyRate analysis'
)
return selected_ifg_files
def main(params):
"""
Parse parameters and prepare files for conversion.
:param dict params: Parameters dictionary read in from the config file
"""
# TODO: looks like base_ifg_paths are ordered according to ifg list
# This probably won't be a problem because input list won't be reordered
# and the original gamma generated list is ordered) this may not affect
# the important pyrate stuff anyway, but might affect gen_thumbs.py.
# Going to assume base_ifg_paths is ordered correcly
# pylint: disable=too-many-branches
if params[cf.PROCESSOR] == 2: # if geotif
log.warning("'conv2tif' step not required for geotiff!")
return
mpi_vs_multiprocess_logging("conv2tif", params)
base_ifg_paths = params[cf.INTERFEROGRAM_FILES]
if params[cf.COH_FILE_LIST] is not None:
base_ifg_paths.extend(params[cf.COHERENCE_FILE_PATHS])
if params[cf.DEM_FILE] is not None: # optional DEM conversion
base_ifg_paths.append(params[cf.DEM_FILE_PATH])
process_base_ifgs_paths = np.array_split(base_ifg_paths, mpiops.size)[mpiops.rank]
gtiff_paths = do_geotiff(process_base_ifgs_paths, params)
mpiops.comm.barrier()
log.info("Finished 'conv2tif' step")
return gtiff_paths
def _inner(proc_ifgs, phase_data_sum):
if isinstance(proc_ifgs[0], Ifg):
proc_ifgs = proc_ifgs
else:
proc_ifgs = [Ifg(ifg_path) for ifg_path in proc_ifgs]
for ifg in proc_ifgs:
if not ifg.is_open:
ifg.open(readonly=False)
comp = np.isnan(phase_data_sum)
comp = np.ravel(comp, order='F')
if params[cf.PARALLEL]:
phase_data = [i.phase_data for i in proc_ifgs]
log.info("Calculating ref phase using multiprocessing")
ref_phs = Parallel(n_jobs=params[cf.PROCESSES],
verbose=joblib_log_level(cf.LOG_LEVEL))(
delayed(_est_ref_phs_method1)(p, comp)
for p in phase_data)
for n, ifg in enumerate(proc_ifgs):
ifg.phase_data -= ref_phs[n]
else:
log.info("Calculating ref phase")
ref_phs = np.zeros(len(proc_ifgs))
for n, ifg in enumerate(proc_ifgs):
ref_phs[n] = _est_ref_phs_method1(ifg.phase_data, comp)
ifg.phase_data -= ref_phs[n]
for ifg in proc_ifgs:
_update_phase_metadata(ifg)
ifg.close()
return ref_phs
def spatial_low_pass_filter(ts_lp, ifg, params):
"""
Filter time series data spatially using either a Butterworth or Gaussian
low pass filter defined by a cut-off distance. If the cut-off distance is
defined as zero in the parameters dictionary then it is calculated for
each time step using the pyrate.covariance.cvd_from_phase method.
:param ndarray ts_lp: Array of time series data, the result of a temporal
low pass filter operation. shape (ifg.shape, n_epochs)
:param shared.Ifg instance ifg: interferogram object
:param dict params: Dictionary of configuration parameters
:return: ts_hp: filtered time series data of shape (ifg.shape, n_epochs)
:rtype: ndarray
"""
log.info('Applying spatial low-pass filter')
if params[cf.SLPF_NANFILL] == 0:
ts_lp[np.isnan(ts_lp)] = 0 # need it here for cvd and fft
else:
# optionally interpolate, operation is inplace
_interpolate_nans(ts_lp, params[cf.SLPF_NANFILL_METHOD])
r_dist = RDist(ifg)()
for i in range(ts_lp.shape[2]):
ts_lp[:, :, i] = _slpfilter(ts_lp[:, :, i], ifg, r_dist, params)
log.debug('Finished applying spatial low pass filter')
return ts_lp
def maxvar_vcm_calc_wrapper(params):
"""
MPI wrapper for maxvar and vcmt computation
"""
preread_ifgs = params[cf.PREREAD_IFGS]
ifg_paths = [ifg_path.tmp_sampled_path for ifg_path in params[cf.INTERFEROGRAM_FILES]]
log.info('Calculating the temporal variance-covariance matrix')
def _get_r_dist(ifg_path):
"""
Get RDIst class object
"""
ifg = Ifg(ifg_path)
ifg.open()
r_dist = RDist(ifg)()
ifg.close()
return r_dist
r_dist = mpiops.run_once(_get_r_dist, ifg_paths[0])
prcs_ifgs = mpiops.array_split(list(enumerate(ifg_paths)))
process_maxvar = {}
for n, i in prcs_ifgs:
log.debug(f'Calculating maxvar for {n} of process ifgs {len(prcs_ifgs)} of total {len(ifg_paths)}')
process_maxvar[int(n)] = cvd(i, params, r_dist, calc_alpha=True, write_vals=True, save_acg=True)[0]
maxvar_d = shared.join_dicts(mpiops.comm.allgather(process_maxvar))
maxvar = [v[1] for v in sorted(maxvar_d.items(), key=lambda s: s[0])]
vcmt = mpiops.run_once(get_vcmt, preread_ifgs, maxvar)
log.debug("Finished maxvar and vcm calc!")
params[cf.MAXVAR], params[cf.VCMT] = maxvar, vcmt
np.save(Configuration.vcmt_path(params), arr=vcmt)
return maxvar, vcmt
def temporal_low_pass_filter(tsincr, epochlist, params):
"""
Filter time series data temporally using either a Gaussian, triangular
or mean low pass filter defined by a cut-off time period (in years).
:param ndarray tsincr: Array of incremental time series data of shape
(ifg.shape, n_epochs)
:param list epochlist: List of shared.EpochList class instances
:param dict params: Dictionary of configuration parameters
:return: tsfilt_incr: filtered time series data, shape (ifg.shape, nepochs)
:rtype: ndarray
"""
log.info('Applying temporal low-pass filter')
nanmat = ~isnan(tsincr)
tsfilt_incr = np.empty_like(tsincr, dtype=np.float32) * np.nan
intv = np.diff(epochlist.spans) # time interval for the neighboring epoch
span = epochlist.spans[:tsincr.shape[2]] + intv / 2 # accumulated time
rows, cols = tsincr.shape[:2]
cutoff = params[cf.TLPF_CUTOFF]
method = params[cf.TLPF_METHOD]
threshold = params[cf.TLPF_PTHR]
if method == 1: # gaussian filter
func = gauss
elif method == 2: # triangular filter
func = _triangle
else:
func = mean_filter
_tlpfilter(cols, cutoff, nanmat, rows, span, threshold, tsfilt_incr,
tsincr, func)
log.debug("Finished applying temporal low-pass filter")
return tsfilt_incr
def correct_ifgs(config: Configuration) -> None:
"""
Top level function to perform PyRate workflow on given interferograms
"""
params = config.__dict__
__validate_correct_steps(params)
# work out the tiling and add to params dict
_update_params_with_tiles(params)
# create the preread_ifgs dict for use with tiled data
_create_ifg_dict(params)
ifg_paths = [ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]]
# create initial tiled phase_data numpy files on disc
save_numpy_phase(ifg_paths, params)
params[C.REFX_FOUND], params[C.REFY_FOUND] = ref_pixel_calc_wrapper(params)
# run through the correct steps in user specified sequence
for step in params['correct']:
if step == 'phase_closure':
correct_steps[step](params, config)
else:
correct_steps[step](params)
log.info("Finished 'correct' step")
def _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles):
"""
Helper function to obtain time series for spatio-temporal filter
using SVD method
"""
# Is there other existing functions that can perform this same job?
log.info('Calculating time series via SVD method for ' 'APS correction')
# copy params temporarily
new_params = deepcopy(params)
new_params[cf.TIME_SERIES_METHOD] = 2 # use SVD method
process_tiles = mpiops.array_split(tiles)
nvels = None
for t in process_tiles:
log.debug('Calculating time series for tile {} during APS '
'correction'.format(t.index))
ifg_parts = [
shared.IfgPart(p, t, preread_ifgs, params) for p in ifg_paths
]
mst_tile = np.load(
os.path.join(params[cf.TMPDIR], 'mst_mat_{}.npy'.format(t.index)))
tsincr = time_series(ifg_parts, new_params, vcmt=None, mst=mst_tile)[0]
np.save(file=os.path.join(params[cf.TMPDIR],
'tsincr_aps_{}.npy'.format(t.index)),
arr=tsincr)
nvels = tsincr.shape[2]
nvels = mpiops.comm.bcast(nvels, root=0)
mpiops.comm.barrier()
# need to assemble tsincr from all processes
tsincr_g = mpiops.run_once(_assemble_tsincr, ifg_paths, params,
preread_ifgs, tiles, nvels)
log.debug('Finished calculating time series for spatio-temporal filter')
return tsincr_g
def _prepifg_multiprocessing(path, xlooks, ylooks, exts, thresh, crop, params):
"""
Multiprocessing wrapper for prepifg
"""
processor = params[cf.PROCESSOR] # roipac, gamma or geotif
if (processor == GAMMA) or (processor == GEOTIF):
header = gamma.gamma_header(path, params)
elif processor == ROIPAC:
log.info("Warning: ROI_PAC support will be deprecated in a future PyRate release")
header = roipac.roipac_header(path, params)
else:
raise PreprocessError('Processor must be ROI_PAC (0) or GAMMA (1)')
# If we're performing coherence masking, find the coherence file for this IFG.
if params[cf.COH_MASK] and shared._is_interferogram(header):
coherence_path = cf.coherence_paths_for(path, params, tif=True)
coherence_thresh = params[cf.COH_THRESH]
else:
coherence_path = None
coherence_thresh = None
if params[cf.LARGE_TIFS]:
op = output_tiff_filename(path, params[cf.OUT_DIR])
looks_path = cf.mlooked_path(op, ylooks, crop)
return path, coherence_path, looks_path
else:
prepifg_helper.prepare_ifg(path, xlooks, ylooks, exts, thresh, crop, out_path=params[cf.OUT_DIR],
header=header, coherence_path=coherence_path, coherence_thresh=coherence_thresh)
def spatio_temporal_filter(params: dict) -> None:
"""
Applies a spatio-temporal filter to remove the atmospheric phase screen
(APS) and saves the corrected interferograms. Firstly the incremental
time series is computed using the SVD method, before a cascade of temporal
then spatial Gaussian filters is applied. The resulting APS corrections are
saved to disc before being subtracted from each interferogram.
:param params: Dictionary of PyRate configuration parameters.
"""
if params[C.APSEST]:
log.info('Doing APS spatio-temporal filtering')
else:
log.info('APS spatio-temporal filtering not required')
return
tiles = params[C.TILES]
preread_ifgs = params[C.PREREAD_IFGS]
ifg_paths = [
ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]
]
# perform some checks on existing ifgs
log.debug('Checking APS correction status')
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_APS_ERROR):
log.debug('Finished APS correction')
return # return if True condition returned
aps_paths = [MultiplePaths.aps_error_path(i, params) for i in ifg_paths]
if all(a.exists() for a in aps_paths):
log.warning('Reusing APS errors from previous run')
_apply_aps_correction(ifg_paths, aps_paths, params)
return
# obtain the incremental time series using SVD
tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles)
mpiops.comm.barrier()
# get lists of epochs and ifgs
ifgs = list(OrderedDict(sorted(preread_ifgs.items())).values())
epochlist = mpiops.run_once(get_epochs, ifgs)[0]
# first perform temporal high pass filter
ts_hp = temporal_high_pass_filter(tsincr, epochlist, params)
# second perform spatial low pass filter to obtain APS correction in ts domain
ifg = Ifg(ifg_paths[0]) # just grab any for parameters in slpfilter
ifg.open()
ts_aps = spatial_low_pass_filter(ts_hp, ifg, params)
ifg.close()
# construct APS corrections for each ifg
_make_aps_corrections(ts_aps, ifgs, params)
# apply correction to ifgs and save ifgs to disc.
_apply_aps_correction(ifg_paths, aps_paths, params)
# update/save the phase_data in the tiled numpy files
shared.save_numpy_phase(ifg_paths, params)
def remove_orbital_error(ifgs: List, params: dict) -> None:
"""
Wrapper function for PyRate orbital error removal functionality.
NB: the ifg data is modified in situ, rather than create intermediate
files. The network method assumes the given ifgs have already been reduced
to a minimum spanning tree network.
"""
ifg_paths = [i.data_path
for i in ifgs] if isinstance(ifgs[0], Ifg) else ifgs
degree = params[C.ORBITAL_FIT_DEGREE]
method = params[C.ORBITAL_FIT_METHOD]
orbfitlksx = params[C.ORBITAL_FIT_LOOKS_X]
orbfitlksy = params[C.ORBITAL_FIT_LOOKS_Y]
# Sanity check of the orbital params
if type(orbfitlksx) != int or type(orbfitlksy) != int:
msg = f"Multi-look factors for orbital correction should be of type: int"
raise OrbitalError(msg)
if degree not in [PLANAR, QUADRATIC, PART_CUBIC]:
msg = "Invalid degree of %s for orbital correction" % C.ORB_DEGREE_NAMES.get(
degree)
raise OrbitalError(msg)
if method not in [NETWORK_METHOD, INDEPENDENT_METHOD]:
msg = "Invalid method of %s for orbital correction" % C.ORB_METHOD_NAMES.get(
method)
raise OrbitalError(msg)
# Give informative log messages based on selected options
log.info(
f'Calculating {__degrees_as_string(degree)} orbital correction using '
f'{__methods_as_string(method)} method')
if orbfitlksx > 1 or orbfitlksy > 1:
log.info(f'Multi-looking interferograms for orbital correction with '
f'factors of X = {orbfitlksx} and Y = {orbfitlksy}')
if method == INDEPENDENT_METHOD:
iterable_split(independent_orbital_correction, ifg_paths, params)
elif method == NETWORK_METHOD:
# Here we do all the multilooking in one process, but in memory
# could use multiple processes if we write data to disc during
# remove_orbital_error step
# TODO: performance comparison of saving multilooked files on
# disc vs in-memory single-process multilooking
#
# The gdal swig bindings prevent us from doing multi-looking in parallel
# when using multiprocessing because the multilooked ifgs are held in
# memory using in-memory tifs. Parallelism using MPI is possible.
# TODO: Use a flag to select mpi parallel vs multiprocessing in the
# iterable_split function, which will use mpi but can fall back on
# single process based on the flag for the multiprocessing side.
if mpiops.rank == MAIN_PROCESS:
mlooked = __create_multilooked_datasets(params)
_validate_mlooked(mlooked, ifg_paths)
network_orbital_correction(ifg_paths, params, mlooked)
else:
raise OrbitalError("Unrecognised orbital correction method")
def __wrap_closure_check(config: Configuration) -> \
Tuple[
List[str],
NDArray[(Any, Any), Float32],
NDArray[(Any, Any, Any), UInt16],
NDArray[(Any,), UInt16],
List[WeightedLoop]]:
"""
This wrapper function returns the closure check outputs for a single iteration of closure check.
:param config: Configuration class instance
For return variables see docstring in `sum_phase_closures`.
"""
params = config.__dict__
ifg_files = [
ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]
]
ifg_files.sort()
log.debug(f"The number of ifgs in the list is {len(ifg_files)}")
sorted_signed_loops = mpiops.run_once(sort_loops_based_on_weights_and_date,
params)
log.info(
f"Total number of selected closed loops with up to MAX_LOOP_LENGTH = "
f"{params[C.MAX_LOOP_LENGTH]} edges is {len(sorted_signed_loops)}")
if len(sorted_signed_loops) < 1:
return None
retained_loops = mpiops.run_once(discard_loops_containing_max_ifg_count,
sorted_signed_loops, params)
ifgs_with_loops = mpiops.run_once(__drop_ifgs_if_not_part_of_any_loop,
ifg_files, retained_loops, params)
msg = f"After applying MAX_LOOP_REDUNDANCY = {params[C.MAX_LOOP_REDUNDANCY]} criteria, " \
f"{len(retained_loops)} loops are retained"
if len(retained_loops) < 1:
return None
else:
log.info(msg)
closure, ifgs_breach_count, num_occurences_each_ifg = sum_phase_closures(
ifgs_with_loops, retained_loops, params)
if mpiops.rank == 0:
closure_ins = config.closure()
np.save(closure_ins.closure, closure)
np.save(closure_ins.ifgs_breach_count, ifgs_breach_count)
np.save(closure_ins.num_occurences_each_ifg, num_occurences_each_ifg)
np.save(closure_ins.loops, retained_loops, allow_pickle=True)
selected_ifg_files = mpiops.run_once(__drop_ifgs_exceeding_threshold,
ifgs_with_loops, ifgs_breach_count,
num_occurences_each_ifg, params)
# update the ifg list in the parameters dictionary
params[C.INTERFEROGRAM_FILES] = \
mpiops.run_once(update_ifg_list, selected_ifg_files, params[C.INTERFEROGRAM_FILES])
return selected_ifg_files, closure, ifgs_breach_count, num_occurences_each_ifg, retained_loops
def process_ifgs(ifg_paths, params, rows, cols):
"""
Top level function to perform PyRate workflow on given interferograms
:param list ifg_paths: List of interferogram paths
:param dict params: Dictionary of configuration parameters
:param int rows: Number of sub-tiles in y direction
:param int cols: Number of sub-tiles in x direction
:return: refpt: tuple of reference pixel x and y position
:rtype: tuple
:return: maxvar: array of maximum variance values of interferograms
:rtype: ndarray
:return: vcmt: Variance-covariance matrix array
:rtype: ndarray
"""
if mpiops.size > 1: # turn of multiprocessing during mpi jobs
params[cf.PARALLEL] = False
outdir = params[cf.TMPDIR]
if not os.path.exists(outdir):
shared.mkdir_p(outdir)
tiles = mpiops.run_once(get_tiles, ifg_paths[0], rows, cols)
preread_ifgs = _create_ifg_dict(ifg_paths, params=params)
# validate user supplied ref pixel
refpixel.validate_supplied_lat_lon(params)
refpx, refpy = _ref_pixel_calc(ifg_paths, params)
# remove non ifg keys
_ = [preread_ifgs.pop(k) for k in ['gt', 'epochlist', 'md', 'wkt']]
multi_paths = params[cf.INTERFEROGRAM_FILES]
_orb_fit_calc(multi_paths, params, preread_ifgs)
_ref_phase_estimation(ifg_paths, params, refpx, refpy)
shared.save_numpy_phase(ifg_paths, tiles, params)
_mst_calc(ifg_paths, params, tiles, preread_ifgs)
# spatio-temporal aps filter
wrap_spatio_temporal_filter(ifg_paths, params, tiles, preread_ifgs)
maxvar, vcmt = _maxvar_vcm_calc(ifg_paths, params, preread_ifgs)
# save phase data tiles as numpy array for timeseries and stackrate calc
shared.save_numpy_phase(ifg_paths, tiles, params)
_timeseries_calc(ifg_paths, params, vcmt, tiles, preread_ifgs)
_stack_calc(ifg_paths, params, vcmt, tiles, preread_ifgs)
log.info('PyRate workflow completed')
return (refpx, refpy), maxvar, vcmt
def independent_orbital_correction(ifg, params):
"""
Calculates and removes an orbital error surface from a single independent
interferogram.
Warning: This will write orbital error corrected phase_data to the ifg.
:param Ifg class instance ifg: the interferogram to be corrected
:param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC)
:param bool offset: True to calculate the model using an offset
:param dict params: dictionary of configuration parameters
:return: None - interferogram phase data is updated and saved to disk
"""
degree = params[cf.ORBITAL_FIT_DEGREE]
offset = params[cf.ORBFIT_OFFSET]
orbfit_correction_on_disc = MultiplePaths.orb_error_path(
ifg.data_path, params)
if not ifg.is_open:
ifg.open()
shared.nan_and_mm_convert(ifg, params)
if orbfit_correction_on_disc.exists():
log.info(
f'Reusing already computed orbital fit correction for {ifg.data_path}'
)
orbital_correction = np.load(file=orbfit_correction_on_disc)
else:
# vectorise, keeping NODATA
vphase = reshape(ifg.phase_data, ifg.num_cells)
dm = get_design_matrix(ifg, degree, offset)
# filter NaNs out before getting model
clean_dm = dm[~isnan(vphase)]
data = vphase[~isnan(vphase)]
model = lstsq(clean_dm, data)[0] # first arg is the model params
# calculate forward model & morph back to 2D
if offset:
fullorb = np.reshape(np.dot(dm[:, :-1], model[:-1]),
ifg.phase_data.shape)
else:
fullorb = np.reshape(np.dot(dm, model), ifg.phase_data.shape)
if not orbfit_correction_on_disc.parent.exists():
shared.mkdir_p(orbfit_correction_on_disc.parent)
offset_removal = nanmedian(np.ravel(ifg.phase_data - fullorb))
orbital_correction = fullorb - offset_removal
# dump to disc
np.save(file=orbfit_correction_on_disc, arr=orbital_correction)
# subtract orbital error from the ifg
ifg.phase_data -= orbital_correction
# set orbfit meta tag and save phase to file
_save_orbital_error_corrected_phase(ifg)
ifg.close()
def __reuse_ref_pixel_file_if_exists():
if ref_pixel_file.exists():
refx, refy = np.load(ref_pixel_file)
log.info('Reusing pre-calculated ref-pixel values: ({}, {}) from file {}'.format(
refx, refy, ref_pixel_file.as_posix()))
log.warning("Reusing ref-pixel values from previous run!!!")
params[C.REFX_FOUND], params[C.REFY_FOUND] = int(refx), int(refy)
return int(refx), int(refy)
else:
return None, None
def _copy_mlooked(params):
log.info(
"Copying input files into tempdir for manipulation during 'correct' steps"
)
mpaths = params[cf.INTERFEROGRAM_FILES]
process_mpaths = mpiops.array_split(mpaths)
for p in process_mpaths:
shutil.copy(p.sampled_path, p.tmp_sampled_path)
Path(p.tmp_sampled_path).chmod(
0o664) # assign write permission as prepifg output is readonly
