def maxvar_vcm_calc_wrapper(params):
"""
MPI wrapper for maxvar and vcmt computation
"""
preread_ifgs = params[cf.PREREAD_IFGS]
ifg_paths = [ifg_path.tmp_sampled_path for ifg_path in params[cf.INTERFEROGRAM_FILES]]
log.info('Calculating the temporal variance-covariance matrix')
def _get_r_dist(ifg_path):
"""
Get RDIst class object
"""
ifg = Ifg(ifg_path)
ifg.open()
r_dist = RDist(ifg)()
ifg.close()
return r_dist
r_dist = mpiops.run_once(_get_r_dist, ifg_paths[0])
prcs_ifgs = mpiops.array_split(list(enumerate(ifg_paths)))
process_maxvar = {}
for n, i in prcs_ifgs:
log.debug(f'Calculating maxvar for {n} of process ifgs {len(prcs_ifgs)} of total {len(ifg_paths)}')
process_maxvar[int(n)] = cvd(i, params, r_dist, calc_alpha=True, write_vals=True, save_acg=True)[0]
maxvar_d = shared.join_dicts(mpiops.comm.allgather(process_maxvar))
maxvar = [v[1] for v in sorted(maxvar_d.items(), key=lambda s: s[0])]
vcmt = mpiops.run_once(get_vcmt, preread_ifgs, maxvar)
log.debug("Finished maxvar and vcm calc!")
params[cf.MAXVAR], params[cf.VCMT] = maxvar, vcmt
np.save(Configuration.vcmt_path(params), arr=vcmt)
return maxvar, vcmt
def test_vcm_legacy_vs_mpi(mpisync, tempdir, roipac_or_gamma_conf):
params = configuration.Configuration(roipac_or_gamma_conf).__dict__
LEGACY_VCM_DIR = os.path.join(SML_TEST_DIR, 'vcm')
legacy_vcm = np.genfromtxt(os.path.join(LEGACY_VCM_DIR, 'vcmt.csv'), delimiter=',')
tmpdir = Path(mpiops.run_once(tempdir))
mpiops.run_once(common.copytree, params[cf.OBS_DIR], tmpdir)
params[cf.OUT_DIR] = tmpdir.joinpath('out')
params[cf.PARALLEL] = 0
output_conf = Path(tmpdir).joinpath('conf.cfg')
cf.write_config_file(params=params, output_conf_file=output_conf)
params = configuration.Configuration(output_conf).__dict__
dest_paths = [p.sampled_path for p in params[cf.INTERFEROGRAM_FILES]]
# run conv2tif and prepifg, create the dest_paths files
conv2tif.main(params)
params[cf.INTERFEROGRAM_FILES].pop()
prepifg.main(params)
params[cf.INTERFEROGRAM_FILES].pop()
preread_ifgs = process._create_ifg_dict(dest_paths, params=params)
refpx, refpy = process._ref_pixel_calc(dest_paths, params)
process._orb_fit_calc(params[cf.INTERFEROGRAM_FILES], params)
process._ref_phase_estimation(dest_paths, params, refpx, refpy)
maxvar, vcmt = process._maxvar_vcm_calc(dest_paths, params, preread_ifgs)
# phase data after ref pixel has changed due to commit bf2f7ebd
# Legacy tests won't match anymore
np.testing.assert_array_almost_equal(maxvar, legacy_maxvar, decimal=4)
np.testing.assert_array_almost_equal(legacy_vcm, vcmt, decimal=3)
mpiops.run_once(shutil.rmtree, tmpdir)
def test_vcm_legacy_vs_mpi(mpisync, tempdir, roipac_or_gamma_conf):
params = configuration.Configuration(roipac_or_gamma_conf).__dict__
LEGACY_VCM_DIR = os.path.join(SML_TEST_DIR, 'vcm')
legacy_vcm = np.genfromtxt(os.path.join(LEGACY_VCM_DIR, 'vcmt.csv'),
delimiter=',')
tmpdir = Path(mpiops.run_once(tempdir))
mpiops.run_once(common.copytree, params[cf.OBS_DIR], tmpdir)
params[cf.OUT_DIR] = tmpdir.joinpath('out')
params[cf.PARALLEL] = False
xlks, ylks, crop = cf.transform_params(params)
base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST],
params[cf.OBS_DIR])
# dest_paths are tifs that have been geotif converted and multilooked
dest_paths = cf.get_dest_paths(base_unw_paths, crop, params, xlks)
# run conv2tif and prepifg, create the dest_paths files
conv2tif.main(params)
prepifg.main(params)
tiles = pyrate.core.shared.get_tiles(dest_paths[0], rows=1, cols=1)
preread_ifgs = process._create_ifg_dict(dest_paths,
params=params,
tiles=tiles)
refpx, refpy = process._ref_pixel_calc(dest_paths, params)
process._orb_fit_calc(dest_paths, params)
process._ref_phase_estimation(dest_paths, params, refpx, refpy)
maxvar, vcmt = process._maxvar_vcm_calc(dest_paths, params, preread_ifgs)
np.testing.assert_array_almost_equal(maxvar, legacy_maxvar, decimal=4)
np.testing.assert_array_almost_equal(legacy_vcm, vcmt, decimal=3)
mpiops.run_once(shutil.rmtree, tmpdir)
def test_vcm_legacy_vs_mpi(mpisync, tempdir, roipac_or_gamma_conf):
params = configuration.Configuration(roipac_or_gamma_conf).__dict__
LEGACY_VCM_DIR = os.path.join(SML_TEST_DIR, 'vcm')
legacy_vcm = np.genfromtxt(os.path.join(LEGACY_VCM_DIR, 'vcmt.csv'),
delimiter=',')
tmpdir = Path(mpiops.run_once(tempdir))
params[C.OUT_DIR] = tmpdir.joinpath('out')
params[C.PARALLEL] = 0
output_conf = Path(tmpdir).joinpath('conf.cfg')
pyrate.configuration.write_config_file(params=params,
output_conf_file=output_conf)
params = configuration.Configuration(output_conf).__dict__
# dest_paths = [p.sampled_path for p in params[cf.INTERFEROGRAM_FILES]]
# run conv2tif and prepifg, create the dest_paths files
conv2tif.main(params)
params = configuration.Configuration(output_conf).__dict__
prepifg.main(params)
params = configuration.Configuration(output_conf).__dict__
params[C.ORBFIT_OFFSET] = True
correct._copy_mlooked(params=params)
correct._update_params_with_tiles(params)
correct._create_ifg_dict(params=params)
pyrate.core.refpixel.ref_pixel_calc_wrapper(params)
pyrate.core.orbital.orb_fit_calc_wrapper(params)
pyrate.core.ref_phs_est.ref_phase_est_wrapper(params)
maxvar, vcmt = pyrate.core.covariance.maxvar_vcm_calc_wrapper(params)
# phase data after ref pixel has changed due to commit bf2f7ebd
# Legacy tests won't match anymore
np.testing.assert_array_almost_equal(maxvar, legacy_maxvar, decimal=4)
np.testing.assert_array_almost_equal(legacy_vcm, vcmt, decimal=3)
mpiops.run_once(shutil.rmtree, tmpdir)
def spatio_temporal_filter(params: dict) -> None:
"""
Applies a spatio-temporal filter to remove the atmospheric phase screen
(APS) and saves the corrected interferograms. Firstly the incremental
time series is computed using the SVD method, before a cascade of temporal
then spatial Gaussian filters is applied. The resulting APS corrections are
saved to disc before being subtracted from each interferogram.
:param params: Dictionary of PyRate configuration parameters.
"""
if params[C.APSEST]:
log.info('Doing APS spatio-temporal filtering')
else:
log.info('APS spatio-temporal filtering not required')
return
tiles = params[C.TILES]
preread_ifgs = params[C.PREREAD_IFGS]
ifg_paths = [
ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]
]
# perform some checks on existing ifgs
log.debug('Checking APS correction status')
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_APS_ERROR):
log.debug('Finished APS correction')
return # return if True condition returned
aps_paths = [MultiplePaths.aps_error_path(i, params) for i in ifg_paths]
if all(a.exists() for a in aps_paths):
log.warning('Reusing APS errors from previous run')
_apply_aps_correction(ifg_paths, aps_paths, params)
return
# obtain the incremental time series using SVD
tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles)
mpiops.comm.barrier()
# get lists of epochs and ifgs
ifgs = list(OrderedDict(sorted(preread_ifgs.items())).values())
epochlist = mpiops.run_once(get_epochs, ifgs)[0]
# first perform temporal high pass filter
ts_hp = temporal_high_pass_filter(tsincr, epochlist, params)
# second perform spatial low pass filter to obtain APS correction in ts domain
ifg = Ifg(ifg_paths[0]) # just grab any for parameters in slpfilter
ifg.open()
ts_aps = spatial_low_pass_filter(ts_hp, ifg, params)
ifg.close()
# construct APS corrections for each ifg
_make_aps_corrections(ts_aps, ifgs, params)
# apply correction to ifgs and save ifgs to disc.
_apply_aps_correction(ifg_paths, aps_paths, params)
# update/save the phase_data in the tiled numpy files
shared.save_numpy_phase(ifg_paths, params)
def __wrap_closure_check(config: Configuration) -> \
Tuple[
List[str],
NDArray[(Any, Any), Float32],
NDArray[(Any, Any, Any), UInt16],
NDArray[(Any,), UInt16],
List[WeightedLoop]]:
"""
This wrapper function returns the closure check outputs for a single iteration of closure check.
:param config: Configuration class instance
For return variables see docstring in `sum_phase_closures`.
"""
params = config.__dict__
ifg_files = [
ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]
]
ifg_files.sort()
log.debug(f"The number of ifgs in the list is {len(ifg_files)}")
sorted_signed_loops = mpiops.run_once(sort_loops_based_on_weights_and_date,
params)
log.info(
f"Total number of selected closed loops with up to MAX_LOOP_LENGTH = "
f"{params[C.MAX_LOOP_LENGTH]} edges is {len(sorted_signed_loops)}")
if len(sorted_signed_loops) < 1:
return None
retained_loops = mpiops.run_once(discard_loops_containing_max_ifg_count,
sorted_signed_loops, params)
ifgs_with_loops = mpiops.run_once(__drop_ifgs_if_not_part_of_any_loop,
ifg_files, retained_loops, params)
msg = f"After applying MAX_LOOP_REDUNDANCY = {params[C.MAX_LOOP_REDUNDANCY]} criteria, " \
f"{len(retained_loops)} loops are retained"
if len(retained_loops) < 1:
return None
else:
log.info(msg)
closure, ifgs_breach_count, num_occurences_each_ifg = sum_phase_closures(
ifgs_with_loops, retained_loops, params)
if mpiops.rank == 0:
closure_ins = config.closure()
np.save(closure_ins.closure, closure)
np.save(closure_ins.ifgs_breach_count, ifgs_breach_count)
np.save(closure_ins.num_occurences_each_ifg, num_occurences_each_ifg)
np.save(closure_ins.loops, retained_loops, allow_pickle=True)
selected_ifg_files = mpiops.run_once(__drop_ifgs_exceeding_threshold,
ifgs_with_loops, ifgs_breach_count,
num_occurences_each_ifg, params)
# update the ifg list in the parameters dictionary
params[C.INTERFEROGRAM_FILES] = \
mpiops.run_once(update_ifg_list, selected_ifg_files, params[C.INTERFEROGRAM_FILES])
return selected_ifg_files, closure, ifgs_breach_count, num_occurences_each_ifg, retained_loops
def dem_error_calc_wrapper(params: dict) -> None:
"""
MPI wrapper for DEM error correction
:param params: Dictionary of PyRate configuration parameters.
"""
if not params[C.DEMERROR]:
log.info("DEM error correction not required")
return
# geometry information needed to calculate Bperp for each pixel using first IFG in list
ifg_paths = [
ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]
]
# check if DEM error correction is already available
if mpiops.run_once(__check_and_apply_demerrors_found_on_disc, ifg_paths,
params):
log.warning("Reusing DEM error correction from previous run!!!")
else:
log.info("Calculating DEM error correction")
# read and open the first IFG in list
ifg0_path = ifg_paths[0]
ifg0 = Ifg(ifg0_path)
ifg0.open(readonly=True)
# not currently implemented for ROIPAC data which breaks some tests
# if statement can be deleted once ROIPAC is deprecated from PyRate
if ifg0.meta_data[ifc.PYRATE_INSAR_PROCESSOR] == 'ROIPAC':
log.warning(
"Geometry calculations are not implemented for ROI_PAC")
return
if params[C.BASE_FILE_LIST] is None:
log.warning(
"No baseline files supplied: DEM error has not been computed")
return
# todo: subtract other corrections (e.g. orbital) from displacement phase before estimating the DEM error
# the following code is a quick way to do the bperp calculation, but is not identical to the GAMMA output
# where the near range of the first SLC is used for each pair.
# calculate look angle for interferograms (using the Near Range of the first SLC)
# look_angle = geometry.calc_local_geometry(ifg0, None, rg, lon, lat, params)
# bperp = geometry.calc_local_baseline(ifg0, az, look_angle)
# split full arrays in to tiles for parallel processing
tiles_split(_process_dem_error_per_tile, params)
preread_ifgs = params[C.PREREAD_IFGS]
# write dem error and correction values to file
mpiops.run_once(_write_dem_errors, ifg_paths, params, preread_ifgs)
shared.save_numpy_phase(ifg_paths, params)
log.debug('Finished DEM error correction step')
def main(params):
"""
PyRate merge main function. Assembles product tiles in to
single geotiff files
"""
# setup paths
rows, cols = params["rows"], params["cols"]
mpiops.run_once(_merge_stack, rows, cols, params)
mpiops.run_once(_create_png_from_tif, params[cf.OUT_DIR])
if params[cf.TIME_SERIES_CAL]:
_merge_timeseries(rows, cols, params)
def _maxvar_vcm_calc(ifg_paths, params, preread_ifgs):
"""
MPI wrapper for maxvar and vcmt computation
"""
log.info('Calculating the temporal variance-covariance matrix')
process_indices = mpiops.array_split(range(len(ifg_paths)))
def _get_r_dist(ifg_path):
"""
Get RDIst class object
"""
ifg = Ifg(ifg_path)
ifg.open()
r_dist = vcm_module.RDist(ifg)()
ifg.close()
return r_dist
r_dist = mpiops.run_once(_get_r_dist, ifg_paths[0])
prcs_ifgs = mpiops.array_split(ifg_paths)
process_maxvar = []
for n, i in enumerate(prcs_ifgs):
log.debug(
'Calculating maxvar for {} of process ifgs {} of total {}'.format(
n + 1, len(prcs_ifgs), len(ifg_paths)))
process_maxvar.append(
vcm_module.cvd(i,
params,
r_dist,
calc_alpha=True,
write_vals=True,
save_acg=True)[0])
if mpiops.rank == MASTER_PROCESS:
maxvar = np.empty(len(ifg_paths), dtype=np.float64)
maxvar[process_indices] = process_maxvar
for i in range(1, mpiops.size): # pragma: no cover
rank_indices = mpiops.array_split(range(len(ifg_paths)), i)
this_process_ref_phs = np.empty(len(rank_indices),
dtype=np.float64)
mpiops.comm.Recv(this_process_ref_phs, source=i, tag=i)
maxvar[rank_indices] = this_process_ref_phs
else: # pragma: no cover
maxvar = np.empty(len(ifg_paths), dtype=np.float64)
mpiops.comm.Send(np.array(process_maxvar, dtype=np.float64),
dest=MASTER_PROCESS,
tag=mpiops.rank)
mpiops.comm.barrier()
maxvar = mpiops.comm.bcast(maxvar, root=0)
vcmt = mpiops.run_once(vcm_module.get_vcmt, preread_ifgs, maxvar)
log.debug("Finished maxvar and vcm calc!")
return maxvar, vcmt
def main(params: dict) -> None:
"""
PyRate merge main function. Assembles product tiles in to
single geotiff files
"""
if params[C.SIGNAL_POLARITY] == 1:
log.info(
f"Saving output products with the same sign convention as input data"
)
elif params[C.SIGNAL_POLARITY] == -1:
log.info(
f"Saving output products with reversed sign convention compared to the input data"
)
else:
log.warning(f"Check the value of signal_polarity parameter")
out_types = []
tsfile = join(params[C.TMPDIR], 'tscuml_0.npy')
if exists(tsfile):
_merge_timeseries(params, 'tscuml')
_merge_linrate(params)
out_types += ['linear_rate', 'linear_error', 'linear_rsquared']
# optional save of merged tsincr products
if params["savetsincr"] == 1:
_merge_timeseries(params, 'tsincr')
else:
log.warning(
'Not merging time series products; {} does not exist'.format(
tsfile))
stfile = join(params[C.TMPDIR], 'stack_rate_0.npy')
if exists(stfile):
# setup paths
mpiops.run_once(_merge_stack, params)
out_types += ['stack_rate', 'stack_error']
else:
log.warning(
'Not merging stack products; {} does not exist'.format(stfile))
if len(out_types) > 0:
process_out_types = mpiops.array_split(out_types)
for out_type in process_out_types:
create_png_and_kml_from_tif(params[C.VELOCITY_DIR],
output_type=out_type)
else:
log.warning('Exiting: no products to merge')
def _merge_linrate(params: dict) -> None:
"""
Merge linear rate outputs
"""
shape, tiles, ifgs_dict = mpiops.run_once(__merge_setup, params)
log.info('Merging and writing Linear Rate product geotiffs')
# read and assemble tile outputs
out_types = [
'linear_' + x
for x in ['rate', 'rsquared', 'error', 'intercept', 'samples']
]
process_out_types = mpiops.array_split(out_types)
for p_out_type in process_out_types:
out = assemble_tiles(shape,
params[C.TMPDIR],
tiles,
out_type=p_out_type)
__save_merged_files(ifgs_dict,
params,
out,
p_out_type,
savenpy=params["savenpy"])
mpiops.comm.barrier()
def spatio_temporal_filter(tsincr, ifg_paths, params, preread_ifgs):
"""
Applies a spatio-temporal filter to remove the atmospheric phase screen
(APS) and saves the corrected interferograms. Before performing this step,
the time series is computed using the SVD method. This function then
performs temporal and spatial filtering.
:param ndarray tsincr: incremental time series array of size
(ifg.shape, nepochs-1)
:param list ifg: List of pyrate.shared.Ifg class objects.
:param dict params: Dictionary of configuration parameter
:param list tiles: List of pyrate.shared.Tile class objects
:param dict preread_ifgs: Dictionary of shared.PrereadIfg class instances
:return: None, corrected interferograms are saved to disk
"""
ifg = Ifg(ifg_paths[0]) # just grab any for parameters in slpfilter
ifg.open()
epochlist = mpiops.run_once(get_epochs, preread_ifgs)[0]
ts_lp = temporal_low_pass_filter(tsincr, epochlist, params)
ts_hp = tsincr - ts_lp
ts_aps = spatial_low_pass_filter(ts_hp, ifg, params)
tsincr -= ts_aps
_ts_to_ifgs(tsincr, preread_ifgs, params)
ifg.close()
def _ts_to_ifgs(tsincr, preread_ifgs, params):
"""
Function that converts an incremental displacement time series into
interferometric phase observations. Used to re-construct an interferogram
network from a time series.
:param ndarray tsincr: incremental time series array of size
(ifg.shape, nepochs-1)
:param dict preread_ifgs: Dictionary of shared.PrereadIfg class instances
:return: None, interferograms are saved to disk
"""
log.debug('Reconstructing interferometric observations from time series')
ifgs = list(OrderedDict(sorted(preread_ifgs.items())).values())
_, n = mpiops.run_once(get_epochs, ifgs)
index_first, index_second = n[:len(ifgs)], n[len(ifgs):]
num_ifgs_tuples = mpiops.array_split(list(enumerate(ifgs)))
num_ifgs_tuples = [(int(num), ifg) for num, ifg in num_ifgs_tuples]
for i, ifg in num_ifgs_tuples:
aps_correction_on_disc = MultiplePaths.aps_error_path(
ifg.tmp_path, params)
phase = np.sum(tsincr[:, :, index_first[i]:index_second[i]], axis=2)
np.save(file=aps_correction_on_disc, arr=phase)
_save_aps_corrected_phase(ifg.tmp_path, phase)
def update_refpix_metadata(ifg_paths, refx, refy, transform, params):
"""
Function that adds metadata about the chosen reference pixel to each interferogram.
"""
pyrate_refpix_lon, pyrate_refpix_lat = mpiops.run_once(convert_pixel_value_to_geographic_coordinate, refx, refy, transform)
process_ifgs_paths = mpiops.array_split(ifg_paths)
for ifg_file in process_ifgs_paths:
log.debug("Updating metadata for: "+ifg_file)
ifg = Ifg(ifg_file)
log.debug("Open dataset")
ifg.open(readonly=True)
nan_and_mm_convert(ifg, params)
half_patch_size = params["refchipsize"] // 2
x, y = refx, refy
log.debug("Extract reference pixel windows")
data = ifg.phase_data[y - half_patch_size: y + half_patch_size + 1,
x - half_patch_size: x + half_patch_size + 1]
log.debug("Calculate standard deviation for reference window")
stddev_ref_area = np.nanstd(data)
log.debug("Calculate mean for reference window")
mean_ref_area = np.nanmean(data)
ifg.add_metadata(**{
ifc.PYRATE_REFPIX_X: str(refx),
ifc.PYRATE_REFPIX_Y: str(refy),
ifc.PYRATE_REFPIX_LAT: str(pyrate_refpix_lat),
ifc.PYRATE_REFPIX_LON: str(pyrate_refpix_lon),
ifc.PYRATE_MEAN_REF_AREA: str(mean_ref_area),
ifc.PYRATE_STDDEV_REF_AREA: str(stddev_ref_area)
})
ifg.write_modified_phase()
ifg.close()
def _ref_pixel_calc(ifg_paths, params):
"""
Wrapper for reference pixel calculation
"""
refx = params[cf.REFX]
refy = params[cf.REFY]
ifg = Ifg(ifg_paths[0])
ifg.open(readonly=True)
if refx == -1 or refy == -1:
log.info('Searching for best reference pixel location')
half_patch_size, thresh, grid = refpixel.ref_pixel_setup(
ifg_paths, params)
process_grid = mpiops.array_split(grid)
refpixel.save_ref_pixel_blocks(process_grid, half_patch_size,
ifg_paths, params)
mean_sds = refpixel._ref_pixel_mpi(process_grid, half_patch_size,
ifg_paths, thresh, params)
mean_sds = mpiops.comm.gather(mean_sds, root=0)
if mpiops.rank == MASTER_PROCESS:
mean_sds = np.hstack(mean_sds)
refy, refx = mpiops.run_once(refpixel.find_min_mean, mean_sds, grid)
log.info('Selected reference pixel coordinate: ({}, {})'.format(
refx, refy))
else:
log.info('Reusing reference pixel from config file: ({}, {})'.format(
refx, refy))
ifg.close()
return refx, refy
def main(params):
"""
PyRate merge main function. Assembles product tiles in to
single geotiff files
"""
# setup paths
rows, cols = params["rows"], params["cols"]
_merge_stack(rows, cols, params)
if params[cf.TIME_SERIES_CAL]:
_merge_timeseries(rows, cols, params)
#mpiops.run_once(_delete_tsincr_files, params)
log.info('Creating quicklook images.')
mpiops.run_once(create_png_from_tif, params[cf.OUT_DIR])
log.debug('Finished creating quicklook images.')
def _orb_fit_calc(ifg_paths, params, preread_ifgs=None):
"""
MPI wrapper for orbital fit correction
"""
log.info('Calculating orbital correction')
if not params[cf.ORBITAL_FIT]:
log.info('Orbital correction not required')
return
if preread_ifgs: # don't check except for mpi tests
# perform some general error/sanity checks
log.debug('Checking Orbital error correction status')
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_ORBITAL_ERROR):
log.debug('Finished Orbital error correction')
return # return if True condition returned
if params[cf.ORBITAL_FIT_METHOD] == 1:
prcs_ifgs = mpiops.array_split(ifg_paths)
orbital.remove_orbital_error(prcs_ifgs, params, preread_ifgs)
else:
# Here we do all the multilooking in one process, but in memory
# can use multiple processes if we write data to disc during
# remove_orbital_error step
# A performance comparison should be made for saving multilooked
# files on disc vs in memory single process multilooking
if mpiops.rank == MASTER_PROCESS:
orbital.remove_orbital_error(ifg_paths, params, preread_ifgs)
mpiops.comm.barrier()
log.debug('Finished Orbital error correction')
def _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles):
"""
Helper function to obtain time series for spatio-temporal filter
using SVD method
"""
# Is there other existing functions that can perform this same job?
log.info('Calculating time series via SVD method for ' 'APS correction')
# copy params temporarily
new_params = deepcopy(params)
new_params[cf.TIME_SERIES_METHOD] = 2 # use SVD method
process_tiles = mpiops.array_split(tiles)
nvels = None
for t in process_tiles:
log.debug('Calculating time series for tile {} during APS '
'correction'.format(t.index))
ifg_parts = [
shared.IfgPart(p, t, preread_ifgs, params) for p in ifg_paths
]
mst_tile = np.load(
os.path.join(params[cf.TMPDIR], 'mst_mat_{}.npy'.format(t.index)))
tsincr = time_series(ifg_parts, new_params, vcmt=None, mst=mst_tile)[0]
np.save(file=os.path.join(params[cf.TMPDIR],
'tsincr_aps_{}.npy'.format(t.index)),
arr=tsincr)
nvels = tsincr.shape[2]
nvels = mpiops.comm.bcast(nvels, root=0)
mpiops.comm.barrier()
# need to assemble tsincr from all processes
tsincr_g = mpiops.run_once(_assemble_tsincr, ifg_paths, params,
preread_ifgs, tiles, nvels)
log.debug('Finished calculating time series for spatio-temporal filter')
return tsincr_g
def wrap_spatio_temporal_filter(ifg_paths, params, tiles, preread_ifgs):
"""
A wrapper for the spatio-temporal filter so it can be tested.
See docstring for spatio_temporal_filter.
"""
if params[cf.APSEST]:
log.info('Doing APS spatio-temporal filtering')
else:
log.info('APS spatio-temporal filtering not required')
return
# perform some checks on existing ifgs
log.debug('Checking APS correction status')
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_APS_ERROR):
log.debug('Finished APS correction')
return # return if True condition returned
tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles)
mpiops.comm.barrier()
ifg = Ifg(ifg_paths[0]) # just grab any for parameters in slpfilter
ifg.open()
spatio_temporal_filter(tsincr, ifg, params, preread_ifgs)
ifg.close()
mpiops.comm.barrier()
def process_ifgs(ifg_paths, params, rows, cols):
"""
Top level function to perform PyRate workflow on given interferograms
:param list ifg_paths: List of interferogram paths
:param dict params: Dictionary of configuration parameters
:param int rows: Number of sub-tiles in y direction
:param int cols: Number of sub-tiles in x direction
:return: refpt: tuple of reference pixel x and y position
:rtype: tuple
:return: maxvar: array of maximum variance values of interferograms
:rtype: ndarray
:return: vcmt: Variance-covariance matrix array
:rtype: ndarray
"""
if mpiops.size > 1: # turn of multiprocessing during mpi jobs
params[cf.PARALLEL] = False
outdir = params[cf.TMPDIR]
if not os.path.exists(outdir):
shared.mkdir_p(outdir)
tiles = mpiops.run_once(get_tiles, ifg_paths[0], rows, cols)
preread_ifgs = _create_ifg_dict(ifg_paths, params=params)
# validate user supplied ref pixel
refpixel.validate_supplied_lat_lon(params)
refpx, refpy = _ref_pixel_calc(ifg_paths, params)
# remove non ifg keys
_ = [preread_ifgs.pop(k) for k in ['gt', 'epochlist', 'md', 'wkt']]
multi_paths = params[cf.INTERFEROGRAM_FILES]
_orb_fit_calc(multi_paths, params, preread_ifgs)
_ref_phase_estimation(ifg_paths, params, refpx, refpy)
shared.save_numpy_phase(ifg_paths, tiles, params)
_mst_calc(ifg_paths, params, tiles, preread_ifgs)
# spatio-temporal aps filter
wrap_spatio_temporal_filter(ifg_paths, params, tiles, preread_ifgs)
maxvar, vcmt = _maxvar_vcm_calc(ifg_paths, params, preread_ifgs)
# save phase data tiles as numpy array for timeseries and stackrate calc
shared.save_numpy_phase(ifg_paths, tiles, params)
_timeseries_calc(ifg_paths, params, vcmt, tiles, preread_ifgs)
_stack_calc(ifg_paths, params, vcmt, tiles, preread_ifgs)
log.info('PyRate workflow completed')
return (refpx, refpy), maxvar, vcmt
def remove_orbital_error(ifgs: List, params: dict) -> None:
"""
Wrapper function for PyRate orbital error removal functionality.
NB: the ifg data is modified in situ, rather than create intermediate
files. The network method assumes the given ifgs have already been reduced
to a minimum spanning tree network.
"""
mpiops.run_once(__orb_params_check, params)
ifg_paths = [i.data_path
for i in ifgs] if isinstance(ifgs[0], Ifg) else ifgs
method = params[cf.ORBITAL_FIT_METHOD]
# mlooking is not necessary for independent correction in a computational sense
# can use multiple procesing if write_to_disc=True
if method == INDEPENDENT_METHOD:
log.info('Calculating orbital correction using independent method')
#TODO: implement multi-looking for independent orbit method
if params[cf.ORBITAL_FIT_LOOKS_X] > 1 or params[
cf.ORBITAL_FIT_LOOKS_Y] > 1:
log.warning(
'Multi-looking is not applied in independent orbit method')
ifgs = [shared.Ifg(p)
for p in ifg_paths] if isinstance(ifgs[0], str) else ifgs
process_ifgs = mpiops.array_split(ifgs)
for ifg in process_ifgs:
independent_orbital_correction(ifg, params=params)
elif method == NETWORK_METHOD:
log.info('Calculating orbital correction using network method')
# Here we do all the multilooking in one process, but in memory
# can use multiple processes if we write data to disc during
# remove_orbital_error step
# A performance comparison should be made for saving multilooked
# files on disc vs in memory single process multilooking
if mpiops.rank == MAIN_PROCESS:
mlooked = __create_multilooked_dataset_for_network_correction(
params)
_validate_mlooked(mlooked, ifg_paths)
network_orbital_correction(ifg_paths, params, mlooked)
else:
raise OrbitalError("Unrecognised orbital correction method")
def test_prepifg_mpi(mpisync, get_config, tempdir, roipac_or_gamma, get_lks,
get_crop):
from tests.common import TEST_CONF_ROIPAC, TEST_CONF_GAMMA
from os.path import join, basename
if roipac_or_gamma == 1:
params = get_config(TEST_CONF_GAMMA)
else:
params = get_config(TEST_CONF_ROIPAC)
outdir = mpiops.run_once(tempdir)
params[cf.OUT_DIR] = outdir
params[cf.PARALLEL] = False
params[cf.IFG_LKSX], params[cf.IFG_LKSY] = get_lks, get_lks
params[cf.IFG_CROP_OPT] = get_crop
if roipac_or_gamma == 1:
params[cf.IFG_FILE_LIST] = join(common.SML_TEST_GAMMA, 'ifms_17')
params[cf.OBS_DIR] = common.SML_TEST_GAMMA
params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
params[cf.DEM_HEADER_FILE] = common.SML_TEST_DEM_HDR_GAMMA
conv2tif.main(params)
prepifg.main(params)
common.remove_tifs(params[cf.OBS_DIR])
if mpiops.rank == 0:
if roipac_or_gamma == 1:
params_s = get_config(TEST_CONF_GAMMA)
else:
params_s = get_config(TEST_CONF_ROIPAC)
params_s[cf.OUT_DIR] = tempdir()
params_s[cf.PARALLEL] = True
params_s[cf.IFG_LKSX], params_s[cf.IFG_LKSY] = get_lks, get_lks
params_s[cf.IFG_CROP_OPT] = get_crop
conv2tif.main(params)
if roipac_or_gamma == 1:
base_unw_paths = glob.glob(join(common.SML_TEST_GAMMA,
"*_utm.unw"))
prepifg.main(params)
else:
base_unw_paths = glob.glob(join(common.SML_TEST_OBS, "*.unw"))
prepifg.main(params_s)
mpi_tifs = glob.glob(join(outdir, "*.tif"))
serial_tifs = glob.glob(join(params[cf.OUT_DIR], "*.tif"))
mpi_tifs.sort()
serial_tifs.sort()
# 17 geotifs, and 17 mlooked tifs
assert len(mpi_tifs) == len(serial_tifs)
for m_f, s_f in zip(mpi_tifs, serial_tifs):
assert basename(m_f) == basename(s_f)
shutil.rmtree(outdir)
shutil.rmtree(params_s[cf.OUT_DIR])
common.remove_tifs(params[cf.OBS_DIR])
def process_ifgs(ifg_paths, params, rows, cols):
"""
Top level function to perform PyRate workflow on given interferograms
:param list ifg_paths: List of interferogram paths
:param dict params: Dictionary of configuration parameters
:param int rows: Number of sub-tiles in y direction
:param int cols: Number of sub-tiles in x direction
:return: refpt: tuple of reference pixel x and y position
:rtype: tuple
:return: maxvar: array of maximum variance values of interferograms
:rtype: ndarray
:return: vcmt: Variance-covariance matrix array
:rtype: ndarray
"""
if mpiops.size > 1: # turn of multiprocessing during mpi jobs
params[cf.PARALLEL] = False
tiles = mpiops.run_once(get_tiles, ifg_paths[0], rows, cols)
preread_ifgs = _create_ifg_dict(ifg_paths, params=params, tiles=tiles)
# _mst_calc(ifg_paths, params, tiles, preread_ifgs)
refpx, refpy = _ref_pixel_calc(ifg_paths, params)
log.debug("refpx, refpy: " + str(refpx) + " " + str(refpy))
# remove non ifg keys
_ = [preread_ifgs.pop(k) for k in ['gt', 'epochlist', 'md', 'wkt']]
_orb_fit_calc(ifg_paths, params, preread_ifgs)
_ref_phase_estimation(ifg_paths, params, refpx, refpy)
_mst_calc(ifg_paths, params, tiles, preread_ifgs)
# spatio-temporal aps filter
_wrap_spatio_temporal_filter(ifg_paths, params, tiles, preread_ifgs)
maxvar, vcmt = _maxvar_vcm_calc(ifg_paths, params, preread_ifgs)
# save phase data tiles as numpy array for timeseries and linrate calc
shared.save_numpy_phase(ifg_paths, tiles, params)
_timeseries_calc(ifg_paths, params, vcmt, tiles, preread_ifgs)
_linrate_calc(ifg_paths, params, vcmt, tiles, preread_ifgs)
log.info('PyRate workflow completed')
return (refpx, refpy), maxvar, vcmt
def configure_stage_log(verbosity, step_name, log_file_name='pyrate.log.'):
log_file_name = run_once(str.__add__, log_file_name,
step_name + '.' + datetime.now().isoformat())
ch = MPIStreamHandler()
ch.setLevel(verbosity)
ch.setFormatter(formatter)
fh = logging.FileHandler(log_file_name)
fh.setLevel(verbosity)
fh.setFormatter(formatter)
pyratelogger.addHandler(ch)
pyratelogger.addHandler(fh)
def main(params: dict) -> None:
"""
PyRate merge main function. Assembles product tiles in to
single geotiff files
"""
out_types = []
stfile = join(params[cf.TMPDIR], 'stack_rate_0.npy')
if exists(stfile):
# setup paths
mpiops.run_once(_merge_stack, params)
out_types += ['stack_rate', 'stack_error']
else:
log.warning(
'Not merging stack products; {} does not exist'.format(stfile))
tsfile = join(params[cf.TMPDIR], 'tscuml_0.npy')
if exists(tsfile):
_merge_timeseries(params, 'tscuml')
_merge_linrate(params)
out_types += ['linear_rate', 'linear_error', 'linear_rsquared']
# optional save of merged tsincr products
if params["savetsincr"] == 1:
_merge_timeseries(params, 'tsincr')
else:
log.warning(
'Not merging time series products; {} does not exist'.format(
tsfile))
if len(out_types) > 0:
process_out_types = mpiops.array_split(out_types)
for out_type in process_out_types:
create_png_and_kml_from_tif(params[cf.OUT_DIR],
output_type=out_type)
else:
log.warning('Exiting: no products to merge')
def _ref_phase_estimation(ifg_paths, params, refpx, refpy):
"""
Wrapper for reference phase estimation.
"""
log.info("Calculating reference phase and correcting each interferogram")
if len(ifg_paths) < 2:
raise rpe.ReferencePhaseError(
"At least two interferograms required for reference phase correction ({len_ifg_paths} "
"provided).".format(len_ifg_paths=len(ifg_paths)))
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_REF_PHASE):
log.debug('Finished reference phase correction')
return
if params[cf.REF_EST_METHOD] == 1:
ref_phs = rpe.est_ref_phase_method1(ifg_paths, params)
elif params[cf.REF_EST_METHOD] == 2:
ref_phs = rpe.est_ref_phase_method2(ifg_paths, params, refpx, refpy)
else:
raise rpe.ReferencePhaseError("No such option, use '1' or '2'.")
# Save reference phase numpy arrays to disk.
ref_phs_file = os.path.join(params[cf.TMPDIR], 'ref_phs.npy')
if mpiops.rank == MASTER_PROCESS:
collected_ref_phs = np.zeros(len(ifg_paths), dtype=np.float64)
process_indices = mpiops.array_split(range(len(ifg_paths)))
collected_ref_phs[process_indices] = ref_phs
for r in range(1, mpiops.size):
process_indices = mpiops.array_split(range(len(ifg_paths)), r)
this_process_ref_phs = np.zeros(shape=len(process_indices),
dtype=np.float64)
mpiops.comm.Recv(this_process_ref_phs, source=r, tag=r)
collected_ref_phs[process_indices] = this_process_ref_phs
np.save(file=ref_phs_file, arr=collected_ref_phs)
else:
mpiops.comm.Send(ref_phs, dest=MASTER_PROCESS, tag=mpiops.rank)
log.debug('Finished reference phase correction')
# Preserve old return value so tests don't break.
if isinstance(ifg_paths[0], Ifg):
ifgs = ifg_paths
else:
ifgs = [Ifg(ifg_path) for ifg_path in ifg_paths]
mpiops.comm.barrier()
return ref_phs, ifgs
def phase_closure_wrapper(params: dict, config: Configuration) -> dict:
"""
This wrapper will run the iterative phase closure check to return a stable
list of checked interferograms, and then mask pixels in interferograms that
exceed the unwrapping error threshold.
:param params: Dictionary of PyRate configuration parameters.
:param config: Configuration class instance.
:return: params: Updated dictionary of PyRate configuration parameters.
"""
if not params[C.PHASE_CLOSURE]:
log.info("Phase closure correction is not required!")
return
rets = iterative_closure_check(config)
if rets is None:
log.info("Zero loops are returned from the iterative closure check.")
log.warning("Abandoning phase closure correction without modifying the interferograms.")
return
ifg_files, ifgs_breach_count, num_occurences_each_ifg = rets
# update params with closure checked ifg list
params[C.INTERFEROGRAM_FILES] = \
mpiops.run_once(update_ifg_list, ifg_files, params[C.INTERFEROGRAM_FILES])
if mpiops.rank == 0:
with open(config.phase_closure_filtered_ifgs_list(params), 'w') as f:
lines = [p.converted_path + '\n' for p in params[C.INTERFEROGRAM_FILES]]
f.writelines(lines)
# mask ifgs with nans where phase unwrap threshold is breached
if mpiops.rank == 0:
mask_pixels_with_unwrapping_errors(ifgs_breach_count, num_occurences_each_ifg, params)
_create_ifg_dict(params) # update the preread_ifgs dict
ifg_paths = [ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]]
# update/save the phase_data in the tiled numpy files
save_numpy_phase(ifg_paths, params)
return params
def wrap_spatio_temporal_filter(params):
"""
A wrapper for the spatio-temporal filter so it can be tested.
See docstring for spatio_temporal_filter.
"""
if params[cf.APSEST]:
log.info('Doing APS spatio-temporal filtering')
else:
log.info('APS spatio-temporal filtering not required')
return
tiles = params[cf.TILES]
preread_ifgs = params[cf.PREREAD_IFGS]
ifg_paths = [
ifg_path.tmp_sampled_path
for ifg_path in params[cf.INTERFEROGRAM_FILES]
]
# perform some checks on existing ifgs
log.debug('Checking APS correction status')
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_APS_ERROR):
log.debug('Finished APS correction')
return # return if True condition returned
aps_error_files_on_disc = [
MultiplePaths.aps_error_path(i, params) for i in ifg_paths
]
if all(a.exists() for a in aps_error_files_on_disc):
log.warning("Reusing APS errors from previous run!!!")
for ifg_path, a in mpiops.array_split(
list(zip(ifg_paths, aps_error_files_on_disc))):
phase = np.load(a)
_save_aps_corrected_phase(ifg_path, phase)
else:
tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles)
mpiops.comm.barrier()
spatio_temporal_filter(tsincr, ifg_paths, params, preread_ifgs)
mpiops.comm.barrier()
shared.save_numpy_phase(ifg_paths, params)
def _create_ifg_dict(params):
"""
Save the preread_ifgs dict with information about the ifgs that are
later used for fast loading of Ifg files in IfgPart class
:param list dest_tifs: List of destination tifs
:param dict params: Config dictionary
:param list tiles: List of all Tile instances
:return: preread_ifgs: Dictionary containing information regarding
interferograms that are used later in workflow
:rtype: dict
"""
dest_tifs = [ifg_path for ifg_path in params[C.INTERFEROGRAM_FILES]]
ifgs_dict = {}
process_tifs = mpiops.array_split(dest_tifs)
for d in process_tifs:
ifg = Ifg(d.tmp_sampled_path) # get the writable copy
ifg.open()
nan_and_mm_convert(ifg, params)
ifgs_dict[d.tmp_sampled_path] = PrereadIfg(
path=d.sampled_path,
tmp_path=d.tmp_sampled_path,
nan_fraction=ifg.nan_fraction,
first=ifg.first,
second=ifg.second,
time_span=ifg.time_span,
nrows=ifg.nrows,
ncols=ifg.ncols,
metadata=ifg.meta_data
)
ifg.write_modified_phase() # update phase converted to mm
ifg.close()
ifgs_dict = join_dicts(mpiops.comm.allgather(ifgs_dict))
ifgs_dict = mpiops.run_once(__save_ifgs_dict_with_headers_and_epochs, dest_tifs, ifgs_dict, params, process_tifs)
params[C.PREREAD_IFGS] = ifgs_dict
return ifgs_dict
def _create_ifg_dict(params):
"""
1. Convert ifg phase data into numpy binary files.
2. Save the preread_ifgs dict with information about the ifgs that are
later used for fast loading of Ifg files in IfgPart class
:param list dest_tifs: List of destination tifs
:param dict params: Config dictionary
:param list tiles: List of all Tile instances
:return: preread_ifgs: Dictionary containing information regarding
interferograms that are used later in workflow
:rtype: dict
"""
dest_tifs = [ifg_path for ifg_path in params[cf.INTERFEROGRAM_FILES]]
ifgs_dict = {}
process_tifs = mpiops.array_split(dest_tifs)
for d in process_tifs:
ifg = shared._prep_ifg(d.sampled_path, params)
ifgs_dict[d.tmp_sampled_path] = PrereadIfg(
path=d.sampled_path,
tmp_path=d.tmp_sampled_path,
nan_fraction=ifg.nan_fraction,
first=ifg.first,
second=ifg.second,
time_span=ifg.time_span,
nrows=ifg.nrows,
ncols=ifg.ncols,
metadata=ifg.meta_data)
ifg.close()
ifgs_dict = join_dicts(mpiops.comm.allgather(ifgs_dict))
ifgs_dict = mpiops.run_once(__save_ifgs_dict_with_headers_and_epochs,
dest_tifs, ifgs_dict, params, process_tifs)
params[cf.PREREAD_IFGS] = ifgs_dict
log.debug('Finished converting phase_data to numpy in process {}'.format(
mpiops.rank))
return ifgs_dict
