def correct_ifgs(config: Configuration) -> None:
"""
Top level function to perform PyRate workflow on given interferograms
"""
params = config.__dict__
__validate_correct_steps(params)
# work out the tiling and add to params dict
_update_params_with_tiles(params)
# create the preread_ifgs dict for use with tiled data
_create_ifg_dict(params)
ifg_paths = [ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]]
# create initial tiled phase_data numpy files on disc
save_numpy_phase(ifg_paths, params)
params[C.REFX_FOUND], params[C.REFY_FOUND] = ref_pixel_calc_wrapper(params)
# run through the correct steps in user specified sequence
for step in params['correct']:
if step == 'phase_closure':
correct_steps[step](params, config)
else:
correct_steps[step](params)
log.info("Finished 'correct' step")
def spatio_temporal_filter(params: dict) -> None:
"""
Applies a spatio-temporal filter to remove the atmospheric phase screen
(APS) and saves the corrected interferograms. Firstly the incremental
time series is computed using the SVD method, before a cascade of temporal
then spatial Gaussian filters is applied. The resulting APS corrections are
saved to disc before being subtracted from each interferogram.
:param params: Dictionary of PyRate configuration parameters.
"""
if params[C.APSEST]:
log.info('Doing APS spatio-temporal filtering')
else:
log.info('APS spatio-temporal filtering not required')
return
tiles = params[C.TILES]
preread_ifgs = params[C.PREREAD_IFGS]
ifg_paths = [
ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]
]
# perform some checks on existing ifgs
log.debug('Checking APS correction status')
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_APS_ERROR):
log.debug('Finished APS correction')
return # return if True condition returned
aps_paths = [MultiplePaths.aps_error_path(i, params) for i in ifg_paths]
if all(a.exists() for a in aps_paths):
log.warning('Reusing APS errors from previous run')
_apply_aps_correction(ifg_paths, aps_paths, params)
return
# obtain the incremental time series using SVD
tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles)
mpiops.comm.barrier()
# get lists of epochs and ifgs
ifgs = list(OrderedDict(sorted(preread_ifgs.items())).values())
epochlist = mpiops.run_once(get_epochs, ifgs)[0]
# first perform temporal high pass filter
ts_hp = temporal_high_pass_filter(tsincr, epochlist, params)
# second perform spatial low pass filter to obtain APS correction in ts domain
ifg = Ifg(ifg_paths[0]) # just grab any for parameters in slpfilter
ifg.open()
ts_aps = spatial_low_pass_filter(ts_hp, ifg, params)
ifg.close()
# construct APS corrections for each ifg
_make_aps_corrections(ts_aps, ifgs, params)
# apply correction to ifgs and save ifgs to disc.
_apply_aps_correction(ifg_paths, aps_paths, params)
# update/save the phase_data in the tiled numpy files
shared.save_numpy_phase(ifg_paths, params)
def __run_once(self):
tiles = self.params[cf.TILES]
mst_files = [Configuration.mst_path(self.params, t.index) for t in tiles]
correct._copy_mlooked(self.params)
correct._create_ifg_dict(self.params)
save_numpy_phase(self.ifg_paths, self.params)
mst.mst_calc_wrapper(self.params)
assert all(m.exists() for m in mst_files)
return [os.stat(o).st_mtime for o in mst_files]
def process_ifgs(ifg_paths, params, rows, cols):
"""
Top level function to perform PyRate workflow on given interferograms
:param list ifg_paths: List of interferogram paths
:param dict params: Dictionary of configuration parameters
:param int rows: Number of sub-tiles in y direction
:param int cols: Number of sub-tiles in x direction
:return: refpt: tuple of reference pixel x and y position
:rtype: tuple
:return: maxvar: array of maximum variance values of interferograms
:rtype: ndarray
:return: vcmt: Variance-covariance matrix array
:rtype: ndarray
"""
if mpiops.size > 1: # turn of multiprocessing during mpi jobs
params[cf.PARALLEL] = False
outdir = params[cf.TMPDIR]
if not os.path.exists(outdir):
shared.mkdir_p(outdir)
tiles = mpiops.run_once(get_tiles, ifg_paths[0], rows, cols)
preread_ifgs = _create_ifg_dict(ifg_paths, params=params)
# validate user supplied ref pixel
refpixel.validate_supplied_lat_lon(params)
refpx, refpy = _ref_pixel_calc(ifg_paths, params)
# remove non ifg keys
_ = [preread_ifgs.pop(k) for k in ['gt', 'epochlist', 'md', 'wkt']]
multi_paths = params[cf.INTERFEROGRAM_FILES]
_orb_fit_calc(multi_paths, params, preread_ifgs)
_ref_phase_estimation(ifg_paths, params, refpx, refpy)
shared.save_numpy_phase(ifg_paths, tiles, params)
_mst_calc(ifg_paths, params, tiles, preread_ifgs)
# spatio-temporal aps filter
wrap_spatio_temporal_filter(ifg_paths, params, tiles, preread_ifgs)
maxvar, vcmt = _maxvar_vcm_calc(ifg_paths, params, preread_ifgs)
# save phase data tiles as numpy array for timeseries and stackrate calc
shared.save_numpy_phase(ifg_paths, tiles, params)
_timeseries_calc(ifg_paths, params, vcmt, tiles, preread_ifgs)
_stack_calc(ifg_paths, params, vcmt, tiles, preread_ifgs)
log.info('PyRate workflow completed')
return (refpx, refpy), maxvar, vcmt
def dem_error_calc_wrapper(params: dict) -> None:
"""
MPI wrapper for DEM error correction
:param params: Dictionary of PyRate configuration parameters.
"""
if not params[C.DEMERROR]:
log.info("DEM error correction not required")
return
# geometry information needed to calculate Bperp for each pixel using first IFG in list
ifg_paths = [
ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]
]
# check if DEM error correction is already available
if mpiops.run_once(__check_and_apply_demerrors_found_on_disc, ifg_paths,
params):
log.warning("Reusing DEM error correction from previous run!!!")
else:
log.info("Calculating DEM error correction")
# read and open the first IFG in list
ifg0_path = ifg_paths[0]
ifg0 = Ifg(ifg0_path)
ifg0.open(readonly=True)
# not currently implemented for ROIPAC data which breaks some tests
# if statement can be deleted once ROIPAC is deprecated from PyRate
if ifg0.meta_data[ifc.PYRATE_INSAR_PROCESSOR] == 'ROIPAC':
log.warning(
"Geometry calculations are not implemented for ROI_PAC")
return
if params[C.BASE_FILE_LIST] is None:
log.warning(
"No baseline files supplied: DEM error has not been computed")
return
# todo: subtract other corrections (e.g. orbital) from displacement phase before estimating the DEM error
# the following code is a quick way to do the bperp calculation, but is not identical to the GAMMA output
# where the near range of the first SLC is used for each pair.
# calculate look angle for interferograms (using the Near Range of the first SLC)
# look_angle = geometry.calc_local_geometry(ifg0, None, rg, lon, lat, params)
# bperp = geometry.calc_local_baseline(ifg0, az, look_angle)
# split full arrays in to tiles for parallel processing
tiles_split(_process_dem_error_per_tile, params)
preread_ifgs = params[C.PREREAD_IFGS]
# write dem error and correction values to file
mpiops.run_once(_write_dem_errors, ifg_paths, params, preread_ifgs)
shared.save_numpy_phase(ifg_paths, params)
log.debug('Finished DEM error correction step')
def __run_once(self):
dem_files = [
MultiplePaths.dem_error_path(i, self.params)
for i in self.ifg_paths
]
correct._copy_mlooked(self.params)
correct._update_params_with_tiles(self.params)
correct._create_ifg_dict(self.params)
save_numpy_phase(self.ifg_paths, self.params)
dem_error_calc_wrapper(self.params)
assert all(m.exists() for m in dem_files)
return [os.stat(o).st_mtime for o in dem_files]
def test_calc_dem_errors(self):
# validate output of current version of the code with saved files from an independent test run
# only the reference phase and dem_error are used in this test
# saved dem_error.tif (expected)
dem_error_tif_exp = join(dem_error_path, 'dem_error.tif')
dem = DEM(dem_error_tif_exp)
dem_error_exp = dem.data
# run relevant parts of the 'correct' step
correct._copy_mlooked(self.params)
correct._update_params_with_tiles(self.params)
correct._create_ifg_dict(self.params)
save_numpy_phase(self.ifg_paths, self.params)
# subtract the reference phase to enable comparison with a 'normal' pyrate run
ref_phase_est_wrapper(self.params)
dem_error_calc_wrapper(self.params)
# dem_error.tif from this run (result)
dem_error_tif_res = join(self.params[C.DEM_ERROR_DIR], 'dem_error.tif')
dem = DEM(dem_error_tif_res)
dem_error_res = dem.data
# check equality
np.testing.assert_allclose(dem_error_exp, dem_error_res)
# ifg correction files in subdirectory out/dem_error/
# three different ifgs:
# ifg1 -> short_baseline_ifg: 20180106-20180319 (ca. 3 m)
# ifg2 -> long_baseline_ifg: 20180130-20180412(ca. 108 m)
# ifg3 -> medium_baseline_ifg: 20180412-20180518 (ca. 48 m)
# load saved files
dem_error_ifg1_path = join(dem_error_path,
'20180106-20180319_ifg_20_dem_error.npy')
dem_error_ifg1_exp = np.load(dem_error_ifg1_path)
dem_error_ifg2_path = join(dem_error_path,
'20180130-20180412_ifg_20_dem_error.npy')
dem_error_ifg2_exp = np.load(dem_error_ifg2_path)
dem_error_ifg3_path = join(dem_error_path,
'20180412-20180518_ifg_20_dem_error.npy')
dem_error_ifg3_exp = np.load(dem_error_ifg3_path)
# load correction values saved from this run (result)
dem_error_ifg1_path = Path(self.params[C.DEM_ERROR_DIR]).joinpath(
'20180106-20180319_ifg_20_dem_error.npy')
dem_error_ifg1_res = np.load(dem_error_ifg1_path)
dem_error_ifg2_path = Path(self.params[C.DEM_ERROR_DIR]).joinpath(
'20180130-20180412_ifg_20_dem_error.npy')
dem_error_ifg2_res = np.load(dem_error_ifg2_path)
dem_error_ifg3_path = Path(self.params[C.DEM_ERROR_DIR]).joinpath(
'20180412-20180518_ifg_20_dem_error.npy')
dem_error_ifg3_res = np.load(dem_error_ifg3_path)
# check equality
np.testing.assert_allclose(dem_error_ifg1_exp, dem_error_ifg1_res)
np.testing.assert_allclose(dem_error_ifg2_exp, dem_error_ifg2_res)
np.testing.assert_allclose(dem_error_ifg3_exp, dem_error_ifg3_res)
def orb_fit_calc_wrapper(params: dict) -> None:
"""
MPI wrapper for orbital fit correction
"""
multi_paths = params[cf.INTERFEROGRAM_FILES]
if not params[cf.ORBITAL_FIT]:
log.info('Orbital correction not required!')
return
ifg_paths = [p.tmp_sampled_path for p in multi_paths]
remove_orbital_error(ifg_paths, params)
mpiops.comm.barrier()
shared.save_numpy_phase(ifg_paths, params)
log.debug('Finished Orbital error correction')
def process_ifgs(ifg_paths, params, rows, cols):
"""
Top level function to perform PyRate workflow on given interferograms
:param list ifg_paths: List of interferogram paths
:param dict params: Dictionary of configuration parameters
:param int rows: Number of sub-tiles in y direction
:param int cols: Number of sub-tiles in x direction
:return: refpt: tuple of reference pixel x and y position
:rtype: tuple
:return: maxvar: array of maximum variance values of interferograms
:rtype: ndarray
:return: vcmt: Variance-covariance matrix array
:rtype: ndarray
"""
if mpiops.size > 1: # turn of multiprocessing during mpi jobs
params[cf.PARALLEL] = False
tiles = mpiops.run_once(get_tiles, ifg_paths[0], rows, cols)
preread_ifgs = _create_ifg_dict(ifg_paths, params=params, tiles=tiles)
# _mst_calc(ifg_paths, params, tiles, preread_ifgs)
refpx, refpy = _ref_pixel_calc(ifg_paths, params)
log.debug("refpx, refpy: " + str(refpx) + " " + str(refpy))
# remove non ifg keys
_ = [preread_ifgs.pop(k) for k in ['gt', 'epochlist', 'md', 'wkt']]
_orb_fit_calc(ifg_paths, params, preread_ifgs)
_ref_phase_estimation(ifg_paths, params, refpx, refpy)
_mst_calc(ifg_paths, params, tiles, preread_ifgs)
# spatio-temporal aps filter
_wrap_spatio_temporal_filter(ifg_paths, params, tiles, preread_ifgs)
maxvar, vcmt = _maxvar_vcm_calc(ifg_paths, params, preread_ifgs)
# save phase data tiles as numpy array for timeseries and linrate calc
shared.save_numpy_phase(ifg_paths, tiles, params)
_timeseries_calc(ifg_paths, params, vcmt, tiles, preread_ifgs)
_linrate_calc(ifg_paths, params, vcmt, tiles, preread_ifgs)
log.info('PyRate workflow completed')
return (refpx, refpy), maxvar, vcmt
def _create_ifg_dict(dest_tifs, params, tiles):
"""
1. Convert ifg phase data into numpy binary files.
2. Save the preread_ifgs dict with information about the ifgs that are
later used for fast loading of Ifg files in IfgPart class
:param list dest_tifs: List of destination tifs
:param dict params: Config dictionary
:param list tiles: List of all Tile instances
:return: preread_ifgs: Dictionary containing information regarding
interferograms that are used later in workflow
:rtype: dict
"""
ifgs_dict = {}
process_tifs = mpiops.array_split(dest_tifs)
shared.save_numpy_phase(dest_tifs, tiles, params)
for d in process_tifs:
ifg = shared._prep_ifg(d, params)
ifgs_dict[d] = PrereadIfg(path=d,
nan_fraction=ifg.nan_fraction,
master=ifg.master,
slave=ifg.slave,
time_span=ifg.time_span,
nrows=ifg.nrows,
ncols=ifg.ncols,
metadata=ifg.meta_data)
ifg.close()
ifgs_dict = _join_dicts(mpiops.comm.allgather(ifgs_dict))
preread_ifgs_file = join(params[cf.TMPDIR], 'preread_ifgs.pk')
if mpiops.rank == MASTER_PROCESS:
# add some extra information that's also useful later
gt, md, wkt = shared.get_geotiff_header_info(process_tifs[0])
ifgs_dict['epochlist'] = algorithm.get_epochs(ifgs_dict)[0]
ifgs_dict['gt'] = gt
ifgs_dict['md'] = md
ifgs_dict['wkt'] = wkt
# dump ifgs_dict file for later use
cp.dump(ifgs_dict, open(preread_ifgs_file, 'wb'))
mpiops.comm.barrier()
preread_ifgs = OrderedDict(
sorted(cp.load(open(preread_ifgs_file, 'rb')).items()))
log.debug('Finished converting phase_data to numpy '
'in process {}'.format(mpiops.rank))
return preread_ifgs
def setup_method(cls):
cls.conf = common.TEST_CONF_GAMMA
params = Configuration(cls.conf).__dict__
conv2tif.main(params)
params = Configuration(cls.conf).__dict__
prepifg.main(params)
cls.params = Configuration(cls.conf).__dict__
correct._copy_mlooked(cls.params)
correct._update_params_with_tiles(cls.params)
correct._create_ifg_dict(cls.params)
multi_paths = cls.params[cf.INTERFEROGRAM_FILES]
cls.ifg_paths = [p.tmp_sampled_path for p in multi_paths]
cls.ifgs = [shared.Ifg(i) for i in cls.ifg_paths]
for i in cls.ifgs:
i.open()
shared.save_numpy_phase(cls.ifg_paths, cls.params)
correct.mst_calc_wrapper(cls.params)
def phase_closure_wrapper(params: dict, config: Configuration) -> dict:
"""
This wrapper will run the iterative phase closure check to return a stable
list of checked interferograms, and then mask pixels in interferograms that
exceed the unwrapping error threshold.
:param params: Dictionary of PyRate configuration parameters.
:param config: Configuration class instance.
:return: params: Updated dictionary of PyRate configuration parameters.
"""
if not params[C.PHASE_CLOSURE]:
log.info("Phase closure correction is not required!")
return
rets = iterative_closure_check(config)
if rets is None:
log.info("Zero loops are returned from the iterative closure check.")
log.warning("Abandoning phase closure correction without modifying the interferograms.")
return
ifg_files, ifgs_breach_count, num_occurences_each_ifg = rets
# update params with closure checked ifg list
params[C.INTERFEROGRAM_FILES] = \
mpiops.run_once(update_ifg_list, ifg_files, params[C.INTERFEROGRAM_FILES])
if mpiops.rank == 0:
with open(config.phase_closure_filtered_ifgs_list(params), 'w') as f:
lines = [p.converted_path + '\n' for p in params[C.INTERFEROGRAM_FILES]]
f.writelines(lines)
# mask ifgs with nans where phase unwrap threshold is breached
if mpiops.rank == 0:
mask_pixels_with_unwrapping_errors(ifgs_breach_count, num_occurences_each_ifg, params)
_create_ifg_dict(params) # update the preread_ifgs dict
ifg_paths = [ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]]
# update/save the phase_data in the tiled numpy files
save_numpy_phase(ifg_paths, params)
return params
def wrap_spatio_temporal_filter(params):
"""
A wrapper for the spatio-temporal filter so it can be tested.
See docstring for spatio_temporal_filter.
"""
if params[cf.APSEST]:
log.info('Doing APS spatio-temporal filtering')
else:
log.info('APS spatio-temporal filtering not required')
return
tiles = params[cf.TILES]
preread_ifgs = params[cf.PREREAD_IFGS]
ifg_paths = [
ifg_path.tmp_sampled_path
for ifg_path in params[cf.INTERFEROGRAM_FILES]
]
# perform some checks on existing ifgs
log.debug('Checking APS correction status')
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_APS_ERROR):
log.debug('Finished APS correction')
return # return if True condition returned
aps_error_files_on_disc = [
MultiplePaths.aps_error_path(i, params) for i in ifg_paths
]
if all(a.exists() for a in aps_error_files_on_disc):
log.warning("Reusing APS errors from previous run!!!")
for ifg_path, a in mpiops.array_split(
list(zip(ifg_paths, aps_error_files_on_disc))):
phase = np.load(a)
_save_aps_corrected_phase(ifg_path, phase)
else:
tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles)
mpiops.comm.barrier()
spatio_temporal_filter(tsincr, ifg_paths, params, preread_ifgs)
mpiops.comm.barrier()
shared.save_numpy_phase(ifg_paths, params)
def ref_phase_est_wrapper(params):
"""
Wrapper for reference phase estimation.
"""
ifg_paths = [
ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]
]
refpx, refpy = params[C.REFX_FOUND], params[C.REFY_FOUND]
if len(ifg_paths) < 2:
raise ReferencePhaseError(
"At least two interferograms required for reference phase correction ({len_ifg_paths} "
"provided).".format(len_ifg_paths=len(ifg_paths)))
# this is not going to be true as we now start with fresh multilooked ifg copies - remove?
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_REF_PHASE):
log.warning(
'Reference phase correction already applied to ifgs; returning')
return
ifgs = [Ifg(ifg_path) for ifg_path in ifg_paths]
# Save reference phase numpy arrays to disk.
ref_phs_file = Configuration.ref_phs_file(params)
# If ref phase file exists on disk, then reuse - subtract ref_phase from ifgs and return
if ref_phs_file.exists():
ref_phs = np.load(ref_phs_file)
_update_phase_and_metadata(ifgs, ref_phs, params)
shared.save_numpy_phase(ifg_paths, params)
return ref_phs, ifgs
# determine the reference phase for each ifg
if params[C.REF_EST_METHOD] == 1:
log.info("Calculating reference phase as median of interferogram")
ref_phs = est_ref_phase_ifg_median(ifg_paths, params)
elif params[C.REF_EST_METHOD] == 2:
log.info(
'Calculating reference phase in a patch surrounding pixel (x, y): ({}, {})'
.format(refpx, refpy))
ref_phs = est_ref_phase_patch_median(ifg_paths, params, refpx, refpy)
else:
raise ReferencePhaseError(
"No such option, set parameter 'refest' to '1' or '2'.")
# gather all reference phases from distributed processes and save to disk
if mpiops.rank == MAIN_PROCESS:
collected_ref_phs = np.zeros(len(ifg_paths), dtype=np.float64)
process_indices = mpiops.array_split(range(len(ifg_paths))).astype(
np.uint16)
collected_ref_phs[process_indices] = ref_phs
for r in range(1, mpiops.size):
process_indices = mpiops.array_split(range(len(ifg_paths)),
r).astype(np.uint16)
this_process_ref_phs = np.zeros(shape=len(process_indices),
dtype=np.float64)
mpiops.comm.Recv(this_process_ref_phs, source=r, tag=r)
collected_ref_phs[process_indices] = this_process_ref_phs
np.save(file=ref_phs_file, arr=collected_ref_phs)
else:
collected_ref_phs = np.empty(len(ifg_paths), dtype=np.float64)
mpiops.comm.Send(ref_phs, dest=MAIN_PROCESS, tag=mpiops.rank)
mpiops.comm.Bcast(collected_ref_phs, root=0)
# subtract ref_phase from ifgs
_update_phase_and_metadata(ifgs, collected_ref_phs, params)
mpiops.comm.barrier()
shared.save_numpy_phase(ifg_paths, params)
log.debug("Finished reference phase correction")
# Preserve old return value so tests don't break.
return ref_phs, ifgs
