def update_refpix_metadata(ifg_paths, refx, refy, transform, params):
"""
Function that adds metadata about the chosen reference pixel to each interferogram.
"""
pyrate_refpix_lon, pyrate_refpix_lat = mpiops.run_once(convert_pixel_value_to_geographic_coordinate, refx, refy, transform)
process_ifgs_paths = mpiops.array_split(ifg_paths)
for ifg_file in process_ifgs_paths:
log.debug("Updating metadata for: "+ifg_file)
ifg = Ifg(ifg_file)
log.debug("Open dataset")
ifg.open(readonly=True)
nan_and_mm_convert(ifg, params)
half_patch_size = params["refchipsize"] // 2
x, y = refx, refy
log.debug("Extract reference pixel windows")
data = ifg.phase_data[y - half_patch_size: y + half_patch_size + 1,
x - half_patch_size: x + half_patch_size + 1]
log.debug("Calculate standard deviation for reference window")
stddev_ref_area = np.nanstd(data)
log.debug("Calculate mean for reference window")
mean_ref_area = np.nanmean(data)
ifg.add_metadata(**{
ifc.PYRATE_REFPIX_X: str(refx),
ifc.PYRATE_REFPIX_Y: str(refy),
ifc.PYRATE_REFPIX_LAT: str(pyrate_refpix_lat),
ifc.PYRATE_REFPIX_LON: str(pyrate_refpix_lon),
ifc.PYRATE_MEAN_REF_AREA: str(mean_ref_area),
ifc.PYRATE_STDDEV_REF_AREA: str(stddev_ref_area)
})
ifg.write_modified_phase()
ifg.close()
def pre_prepare_ifgs(ifg_paths, params):
"""
nan and mm convert ifgs
"""
ifgs = [Ifg(p) for p in ifg_paths]
for i in ifgs:
i.open(readonly=False)
nan_and_mm_convert(i, params)
return ifgs
def _write_dem_errors(ifg_paths: list, params: dict,
preread_ifgs: dict) -> None:
"""
Convenience function for writing DEM error (one file) and DEM error correction for each IFG to disc
:param ifg_paths: List of interferogram class objects.
:param params: Dictionary of PyRate configuration parameters.
:param preread_ifgs: Dictionary of interferogram metadata.
"""
tiles = params[C.TILES]
# re-assemble tiles and save into dem_error dir
shape = preread_ifgs[ifg_paths[0]].shape
# save dem error as geotiff file in out directory
gt, md, wkt = shared.get_geotiff_header_info(ifg_paths[0])
md[ifc.DATA_TYPE] = ifc.DEM_ERROR
dem_error = assemble_tiles(shape,
params[C.TMPDIR],
tiles,
out_type='dem_error')
dem_error_file = os.path.join(params[C.DEM_ERROR_DIR], 'dem_error.tif')
shared.remove_file_if_exists(dem_error_file)
shared.write_output_geotiff(md, gt, wkt, dem_error, dem_error_file, np.nan)
# read the average bperp vals for each ifg and each tile
bperp = np.empty(shape=(len(tiles), len(ifg_paths)), dtype=np.float64)
for t in tiles:
bperp_file = Path(
join(params[C.TMPDIR], 'bperp_avg_' + str(t.index) + '.npy'))
arr = np.load(file=bperp_file)
bperp[t.index, :] = arr
# loop over all ifgs
idx = 0
for ifg_path in ifg_paths:
ifg = Ifg(ifg_path)
ifg.open()
shared.nan_and_mm_convert(ifg,
params) # ensure we have phase data in mm
# read dem error correction file from tmpdir
dem_error_correction_ifg = assemble_tiles(
shape,
params[C.TMPDIR],
tiles,
out_type='dem_error_correction',
index=idx)
# calculate average bperp value across all tiles for the ifg
bperp_val = np.nanmean(bperp[:, idx])
dem_error_correction_on_disc = MultiplePaths.dem_error_path(
ifg.data_path, params)
np.save(file=dem_error_correction_on_disc,
arr=dem_error_correction_ifg)
idx += 1
# subtract DEM error from the ifg
ifg.phase_data -= dem_error_correction_ifg
_save_dem_error_corrected_phase(ifg, bperp_val)
def independent_orbital_correction(ifg, params):
"""
Calculates and removes an orbital error surface from a single independent
interferogram.
Warning: This will write orbital error corrected phase_data to the ifg.
:param Ifg class instance ifg: the interferogram to be corrected
:param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC)
:param bool offset: True to calculate the model using an offset
:param dict params: dictionary of configuration parameters
:return: None - interferogram phase data is updated and saved to disk
"""
degree = params[cf.ORBITAL_FIT_DEGREE]
offset = params[cf.ORBFIT_OFFSET]
orbfit_correction_on_disc = MultiplePaths.orb_error_path(
ifg.data_path, params)
if not ifg.is_open:
ifg.open()
shared.nan_and_mm_convert(ifg, params)
if orbfit_correction_on_disc.exists():
log.info(
f'Reusing already computed orbital fit correction for {ifg.data_path}'
)
orbital_correction = np.load(file=orbfit_correction_on_disc)
else:
# vectorise, keeping NODATA
vphase = reshape(ifg.phase_data, ifg.num_cells)
dm = get_design_matrix(ifg, degree, offset)
# filter NaNs out before getting model
clean_dm = dm[~isnan(vphase)]
data = vphase[~isnan(vphase)]
model = lstsq(clean_dm, data)[0] # first arg is the model params
# calculate forward model & morph back to 2D
if offset:
fullorb = np.reshape(np.dot(dm[:, :-1], model[:-1]),
ifg.phase_data.shape)
else:
fullorb = np.reshape(np.dot(dm, model), ifg.phase_data.shape)
if not orbfit_correction_on_disc.parent.exists():
shared.mkdir_p(orbfit_correction_on_disc.parent)
offset_removal = nanmedian(np.ravel(ifg.phase_data - fullorb))
orbital_correction = fullorb - offset_removal
# dump to disc
np.save(file=orbfit_correction_on_disc, arr=orbital_correction)
# subtract orbital error from the ifg
ifg.phase_data -= orbital_correction
# set orbfit meta tag and save phase to file
_save_orbital_error_corrected_phase(ifg)
ifg.close()
def pre_prepare_ifgs(ifg_paths, params):
"""
Open ifg for reading
"""
ifgs = [Ifg(p) for p in ifg_paths]
for i in ifgs:
if not i.is_open:
i.open(readonly=False)
nan_and_mm_convert(i, params)
return ifgs
def __inner(ifg, ref_ph):
ifg.open()
# nan-convert and mm-convert before subtracting ref phase
nan_and_mm_convert(ifg, params)
# add 1e-20 to avoid 0.0 values being converted to NaN downstream (Github issue #310)
# TODO: implement a more robust way of avoiding this issue, e.g. using numpy masked
# arrays to mark invalid pixel values rather than directly changing values to NaN
ifg.phase_data -= ref_ph + 1e-20
ifg.meta_data[ifc.PYRATE_REF_PHASE] = ifc.REF_PHASE_REMOVED
ifg.write_modified_phase()
log.debug(f"Reference phase corrected for {ifg.data_path}")
ifg.close()
def __create_multilooked_datasets(params):
exts, ifg_paths, multi_paths = __extents_from_params(params)
mlooked_datasets = [
_create_mlooked_dataset(m, i, exts, params)
for m, i in zip(multi_paths, ifg_paths)
]
mlooked = [Ifg(m) for m in mlooked_datasets]
for m in mlooked:
m.initialize()
shared.nan_and_mm_convert(m, params)
return mlooked
def cvd(ifg_path,
params,
r_dist,
calc_alpha=False,
write_vals=False,
save_acg=False):
"""
Calculate the 1D covariance function of an entire interferogram as the
radial average of its 2D autocorrelation.
:param str ifg_path: An interferogram file path. OR
:param dict params: Dictionary of configuration parameters
:param ndarray r_dist: Array of distance values from the image centre
(See Rdist class for more details)
:param bool calc_alpha: If True calculate alpha
:param bool write_vals: If True write maxvar and alpha values to
interferogram metadata
:param bool save_acg: If True write autocorrelation and radial distance
data to numpy array file on disk
:return: maxvar: The maximum variance (at zero lag)
:rtype: float
:return: alpha: the exponential length-scale of decay factor
:rtype: float
"""
ifg = shared.Ifg(ifg_path)
ifg.open()
shared.nan_and_mm_convert(ifg, params)
# calculate 2D auto-correlation of image using the
# spectral method (Wiener-Khinchin theorem)
if ifg.nan_converted: # if nancoverted earlier, convert nans back to 0's
phase = where(isnan(ifg.phase_data), 0, ifg.phase_data)
else:
phase = ifg.phase_data
maxvar, alpha = cvd_from_phase(phase,
ifg,
r_dist,
calc_alpha,
save_acg=save_acg,
params=params)
if write_vals:
ifg.add_metadata(**{
ifc.PYRATE_MAXVAR: str(maxvar),
ifc.PYRATE_ALPHA: str(alpha)
})
ifg.close()
return maxvar, alpha
def __plot_ifg(file, cmap, ax, num_ifgs, params):
try:
ifg = Ifg(file)
ifg.open()
except:
raise AttributeError(f'Cannot open interferogram geotiff: {file}')
# change nodata values to NaN for display; convert units to mm (if in radians)
nan_and_mm_convert(ifg, params)
im = ax.imshow(ifg.phase_data, cmap=cmap)
text = ax.set_title(Path(ifg.data_path).stem)
text.set_fontsize(20)
# text.set_fontsize(min(20, int(num_ifgs / 2)))
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.05)
plt.colorbar(im, cax=cax, label='mm')
ifg.close()
def __check_and_apply_orberrors_found_on_disc(ifg_paths, params):
saved_orb_err_paths = [
MultiplePaths.orb_error_path(ifg_path, params)
for ifg_path in ifg_paths
]
for p, i in zip(saved_orb_err_paths, ifg_paths):
if p.exists():
orb = np.load(p)
if isinstance(i, str):
# are paths
ifg = Ifg(i)
ifg.open(readonly=False)
shared.nan_and_mm_convert(ifg, params)
else:
ifg = i
ifg.phase_data -= orb
# set orbfit meta tag and save phase to file
_save_orbital_error_corrected_phase(ifg)
return all(p.exists() for p in saved_orb_err_paths)
def independent_orbital_correction(ifg, degree, offset, params):
"""
Calculates and removes an orbital error surface from a single independent
interferogram.
Warning: This will write orbital error corrected phase_data to the ifg.
:param Ifg class instance ifg: the interferogram to be corrected
:param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC)
:param bool offset: True to calculate the model using an offset
:param dict params: dictionary of configuration parameters
:return: None - interferogram phase data is updated and saved to disk
"""
ifg = shared.Ifg(ifg) if isinstance(ifg, str) else ifg
if not ifg.is_open:
ifg.open()
shared.nan_and_mm_convert(ifg, params)
# vectorise, keeping NODATA
vphase = reshape(ifg.phase_data, ifg.num_cells)
dm = get_design_matrix(ifg, degree, offset)
# filter NaNs out before getting model
clean_dm = dm[~isnan(vphase)]
data = vphase[~isnan(vphase)]
model = lstsq(clean_dm, data)[0] # first arg is the model params
# calculate forward model & morph back to 2D
if offset:
fullorb = np.reshape(np.dot(dm[:, :-1], model[:-1]),
ifg.phase_data.shape)
else:
fullorb = np.reshape(np.dot(dm, model), ifg.phase_data.shape)
offset_removal = nanmedian(np.ravel(ifg.phase_data - fullorb))
# subtract orbital error from the ifg
ifg.phase_data -= (fullorb - offset_removal)
# set orbfit meta tag and save phase to file
_save_orbital_error_corrected_phase(ifg)
if ifg.open():
ifg.close()
def _apply_aps_correction(ifg_paths: List[str], aps_paths: List[str],
params: dict) -> None:
"""
Function to read and apply (subtract) APS corrections from interferogram data.
"""
for ifg_path, aps_path in mpiops.array_split(
list(zip(ifg_paths, aps_paths))):
# read the APS correction from numpy array
aps_corr = np.load(aps_path)
# open the Ifg object
ifg = Ifg(ifg_path)
ifg.open(readonly=False)
# convert NaNs and convert to mm
nan_and_mm_convert(ifg, params)
# subtract the correction from the ifg phase data
ifg.phase_data[~np.isnan(ifg.phase_data
)] -= aps_corr[~np.isnan(ifg.phase_data)]
# set meta-data tags after aps error correction
ifg.dataset.SetMetadataItem(ifc.PYRATE_APS_ERROR, ifc.APS_REMOVED)
# write phase data to disc and close ifg.
ifg.write_modified_phase()
ifg.close()
def _create_ifg_dict(params):
"""
Save the preread_ifgs dict with information about the ifgs that are
later used for fast loading of Ifg files in IfgPart class
:param list dest_tifs: List of destination tifs
:param dict params: Config dictionary
:param list tiles: List of all Tile instances
:return: preread_ifgs: Dictionary containing information regarding
interferograms that are used later in workflow
:rtype: dict
"""
dest_tifs = [ifg_path for ifg_path in params[C.INTERFEROGRAM_FILES]]
ifgs_dict = {}
process_tifs = mpiops.array_split(dest_tifs)
for d in process_tifs:
ifg = Ifg(d.tmp_sampled_path) # get the writable copy
ifg.open()
nan_and_mm_convert(ifg, params)
ifgs_dict[d.tmp_sampled_path] = PrereadIfg(
path=d.sampled_path,
tmp_path=d.tmp_sampled_path,
nan_fraction=ifg.nan_fraction,
first=ifg.first,
second=ifg.second,
time_span=ifg.time_span,
nrows=ifg.nrows,
ncols=ifg.ncols,
metadata=ifg.meta_data
)
ifg.write_modified_phase() # update phase converted to mm
ifg.close()
ifgs_dict = join_dicts(mpiops.comm.allgather(ifgs_dict))
ifgs_dict = mpiops.run_once(__save_ifgs_dict_with_headers_and_epochs, dest_tifs, ifgs_dict, params, process_tifs)
params[C.PREREAD_IFGS] = ifgs_dict
return ifgs_dict
def __create_multilooked_dataset_for_network_correction(params):
multi_paths = params[cf.INTERFEROGRAM_FILES]
ifg_paths = [p.tmp_sampled_path for p in multi_paths]
headers = [find_header(p, params) for p in multi_paths]
crop_opt = prepifg_helper.ALREADY_SAME_SIZE
xlooks = params[cf.ORBITAL_FIT_LOOKS_X]
ylooks = params[cf.ORBITAL_FIT_LOOKS_Y]
thresh = params[cf.NO_DATA_AVERAGING_THRESHOLD]
rasters = [shared.dem_or_ifg(r) for r in ifg_paths]
exts = prepifg_helper.get_analysis_extent(crop_opt, rasters, xlooks,
ylooks, None)
out_paths = [tempfile.mktemp() for _ in ifg_paths]
mlooked_dataset = [
prepifg_helper.prepare_ifg(d, xlooks, ylooks, exts, thresh, crop_opt,
h, False, p)
for d, h, p in zip(ifg_paths, headers, out_paths)
]
mlooked = [Ifg(m[1]) for m in mlooked_dataset]
for m in mlooked:
m.initialize()
shared.nan_and_mm_convert(m, params)
return mlooked
def mask_pixels_with_unwrapping_errors(ifgs_breach_count: NDArray[(
Any, Any, Any), UInt16], num_occurrences_each_ifg: NDArray[(Any, ),
UInt16],
params: dict) -> None:
"""
Find pixels in the phase data that breach closure_thr, and mask
(assign NaNs) to those pixels in those ifgs.
:param ifgs_breach_count: unwrapping issues at pixels in all loops
:param num_occurrences_each_ifg: frequency of ifgs appearing in all loops
:param params: params dict
"""
log.debug("Masking phase data of retained ifgs")
for i, m_p in enumerate(params[C.INTERFEROGRAM_FILES]):
pix_index = ifgs_breach_count[:, :, i] == num_occurrences_each_ifg[i]
ifg = Ifg(m_p.tmp_sampled_path)
ifg.open()
nan_and_mm_convert(ifg, params)
ifg.phase_data[pix_index] = np.nan
ifg.write_modified_phase()
log.info(f"Masked phase data of {i + 1} retained ifgs after phase closure")
return None
def network_orbital_correction(ifg_paths,
params,
m_ifgs: Optional[List] = None):
"""
This algorithm implements a network inversion to determine orbital
corrections for a set of interferograms forming a connected network.
Warning: This will write orbital error corrected phase_data to the ifgs.
:param list ifg_paths: List of Ifg class objects reduced to a minimum spanning
tree network
:param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC)
:param bool offset: True to calculate the model using offsets
:param dict params: dictionary of configuration parameters
:param list m_ifgs: list of multilooked Ifg class objects
(sequence must be multilooked versions of 'ifgs' arg)
:param dict preread_ifgs: Dictionary containing information specifically
for MPI jobs (optional)
:return: None - interferogram phase data is updated and saved to disk
"""
# pylint: disable=too-many-locals, too-many-arguments
offset = params[cf.ORBFIT_OFFSET]
degree = params[cf.ORBITAL_FIT_DEGREE]
preread_ifgs = params[cf.PREREAD_IFGS]
# all orbit corrections available?
if isinstance(ifg_paths[0], str):
if __check_and_apply_orberrors_found_on_disc(ifg_paths, params):
log.warning("Reusing orbfit errors from previous run!!!")
return
# all corrections are available in numpy files already saved - return
ifgs = [shared.Ifg(i) for i in ifg_paths]
else: # alternate test paths # TODO: improve
ifgs = ifg_paths
src_ifgs = ifgs if m_ifgs is None else m_ifgs
src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst
vphase = vstack([i.phase_data.reshape((i.num_cells, 1)) for i in src_ifgs])
vphase = squeeze(vphase)
B = get_network_design_matrix(src_ifgs, degree, offset)
# filter NaNs out before getting model
B = B[~isnan(vphase)]
orbparams = dot(pinv(B, 1e-6), vphase[~isnan(vphase)])
ncoef = _get_num_params(degree)
if preread_ifgs:
temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values()
ids = first_second_ids(get_all_epochs(temp_ifgs))
else:
ids = first_second_ids(get_all_epochs(ifgs))
coefs = [
orbparams[i:i + ncoef] for i in range(0,
len(set(ids)) * ncoef, ncoef)
]
# create full res DM to expand determined coefficients into full res
# orbital correction (eg. expand coarser model to full size)
if preread_ifgs:
temp_ifg = Ifg(ifg_paths[0]) # ifgs here are paths
temp_ifg.open()
dm = get_design_matrix(temp_ifg, degree, offset=False)
temp_ifg.close()
else:
ifg = ifgs[0]
dm = get_design_matrix(ifg, degree, offset=False)
for i in ifg_paths:
# open if not Ifg instance
if isinstance(i, str): # pragma: no cover
# are paths
i = Ifg(i)
i.open(readonly=False)
shared.nan_and_mm_convert(i, params)
_remove_network_orb_error(coefs, dm, i, ids, offset, params)
def independent_orbital_correction(ifg_path, params):
"""
Calculates and removes an orbital error surface from a single independent
interferogram.
Warning: This will write orbital error corrected phase_data to the ifg.
:param Ifg class instance ifg: the interferogram to be corrected
:param dict params: dictionary of configuration parameters
:return: None - interferogram phase data is updated and saved to disk
"""
log.debug(f"Orbital correction of {ifg_path}")
degree = params[C.ORBITAL_FIT_DEGREE]
offset = params[C.ORBFIT_OFFSET]
intercept = params[C.ORBFIT_INTERCEPT]
xlooks = params[C.ORBITAL_FIT_LOOKS_X]
ylooks = params[C.ORBITAL_FIT_LOOKS_Y]
ifg0 = shared.Ifg(ifg_path) if isinstance(ifg_path, str) else ifg_path
# get full-resolution design matrix
fullres_dm = get_design_matrix(ifg0, degree, intercept=intercept)
ifg = shared.dem_or_ifg(ifg_path) if isinstance(ifg_path,
str) else ifg_path
multi_path = MultiplePaths(ifg.data_path, params)
orb_on_disc = MultiplePaths.orb_error_path(ifg.data_path, params)
if not ifg.is_open:
ifg.open()
shared.nan_and_mm_convert(ifg, params)
fullres_ifg = ifg # keep a backup
fullres_phase = fullres_ifg.phase_data
if orb_on_disc.exists():
log.info(
f'Reusing already computed orbital fit correction: {orb_on_disc}')
orbital_correction = np.load(file=orb_on_disc)
else:
# Multi-look the ifg data if either X or Y is greater than 1
if (xlooks > 1) or (ylooks > 1):
exts, _, _ = __extents_from_params(params)
mlooked = _create_mlooked_dataset(multi_path, ifg.data_path, exts,
params)
ifg = Ifg(mlooked) # multi-looked Ifg object
ifg.initialize()
shared.nan_and_mm_convert(ifg, params)
# vectorise phase data, keeping NODATA
vphase = reshape(ifg.phase_data, ifg.num_cells)
# compute design matrix for multi-looked data
mlooked_dm = get_design_matrix(ifg, degree, intercept=intercept)
# invert to obtain the correction image (forward model) at full-res
orbital_correction = __orb_correction(fullres_dm,
mlooked_dm,
fullres_phase,
vphase,
offset=offset)
# save correction to disc
if not orb_on_disc.parent.exists():
shared.mkdir_p(orb_on_disc.parent)
np.save(file=orb_on_disc, arr=orbital_correction)
# subtract orbital correction from the full-res ifg
fullres_ifg.phase_data -= orbital_correction
# set orbfit meta tag and save phase to file
_save_orbital_error_corrected_phase(fullres_ifg, params)
def network_orbital_correction(ifg_paths,
params,
m_ifgs: Optional[List] = None):
"""
This algorithm implements a network inversion to determine orbital
corrections for a set of interferograms forming a connected network.
Warning: This will write orbital error corrected phase_data to the ifgs.
:param list ifg_paths: List of Ifg class objects reduced to a minimum spanning tree network
:param dict params: dictionary of configuration parameters
:param list m_ifgs: list of multilooked Ifg class objects (sequence must be multilooked versions of 'ifgs' arg)
:return: None - interferogram phase data is updated and saved to disk
"""
# pylint: disable=too-many-locals, too-many-arguments
offset = params[C.ORBFIT_OFFSET]
degree = params[C.ORBITAL_FIT_DEGREE]
preread_ifgs = params[C.PREREAD_IFGS]
intercept = params[C.ORBFIT_INTERCEPT]
scale = params[C.ORBFIT_SCALE]
# all orbit corrections available?
if isinstance(ifg_paths[0], str):
if __check_and_apply_orberrors_found_on_disc(ifg_paths, params):
log.warning("Reusing orbfit errors from previous run!!!")
return
# all corrections are available in numpy files already saved - return
ifgs = [shared.Ifg(i) for i in ifg_paths]
else: # alternate test paths # TODO: improve
ifgs = ifg_paths
src_ifgs = ifgs if m_ifgs is None else m_ifgs
src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst
if preread_ifgs:
temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values()
ids = first_second_ids(get_all_epochs(temp_ifgs))
else:
ids = first_second_ids(get_all_epochs(ifgs))
nepochs = len(set(ids))
# call the actual inversion routine
coefs = calc_network_orb_correction(src_ifgs,
degree,
scale,
nepochs,
intercept=intercept)
# create full res DM to expand determined coefficients into full res
# orbital correction (eg. expand coarser model to full size)
if preread_ifgs:
temp_ifg = Ifg(ifg_paths[0]) # ifgs here are paths
temp_ifg.open()
dm = get_design_matrix(temp_ifg,
degree,
intercept=intercept,
scale=scale)
temp_ifg.close()
else:
ifg = ifgs[0]
dm = get_design_matrix(ifg, degree, intercept=intercept, scale=scale)
for i in ifg_paths:
# open if not Ifg instance
if isinstance(i, str): # pragma: no cover
# are paths
i = Ifg(i)
i.open(readonly=False)
shared.nan_and_mm_convert(i, params)
_remove_network_orb_error(coefs, dm, i, ids, offset, params)
