def _geotiff_multiprocessing(unw_path: MultiplePaths,
params: dict) -> Tuple[str, bool]:
"""
Multiprocessing wrapper for full-res geotiff conversion
"""
# TODO: Need a more robust method for identifying coherence files.
dest = unw_path.converted_path
processor = params[C.PROCESSOR] # roipac or gamma
# Create full-res geotiff if not already on disk
if not os.path.exists(dest):
if processor == GAMMA:
header = gamma.gamma_header(unw_path.unwrapped_path, params)
elif processor == ROIPAC:
log.info(
"Warning: ROI_PAC support will be deprecated in a future PyRate release"
)
header = roipac.roipac_header(unw_path.unwrapped_path, params)
else:
raise PreprocessError('Processor must be ROI_PAC (0) or GAMMA (1)')
header[ifc.INPUT_TYPE] = unw_path.input_type
shared.write_fullres_geotiff(header,
unw_path.unwrapped_path,
dest,
nodata=params[C.NO_DATA_VALUE])
Path(dest).chmod(0o444) # readonly output
return dest, True
else:
log.warning(
f"Full-res geotiff already exists in {dest}! Returning existing geotiff!"
)
return dest, False
def convert_to_nans(self):
"""
Convert phase data of given value to NaN
"""
if (self._nodata_value is None) \
or (self.dataset is None): # pragma: no cover
msg = 'nodata value needs to be set for nan conversion.' \
'Use ifg.nodata_value = NoDataValue to set nodata_value'
log.warning(msg)
raise RasterException(msg)
if ((self.dataset.GetMetadataItem(ifc.NAN_STATUS) == ifc.NAN_CONVERTED)
or self.nan_converted):
self.phase_data = self.phase_data
self.nan_converted = True
msg = '{}: ignored as previous nan ' \
'conversion detected'.format(self.data_path)
log.debug(msg)
return
else:
self.phase_data = where(
isclose(self.phase_data, self._nodata_value, atol=1e-6),
nan,
self.phase_data)
self.meta_data[ifc.NAN_STATUS] = ifc.NAN_CONVERTED
self.nan_converted = True
def main(params):
"""
Parse parameters and prepare files for conversion.
:param dict params: Parameters dictionary read in from the config file
"""
# TODO: looks like base_ifg_paths are ordered according to ifg list
# This probably won't be a problem because input list won't be reordered
# and the original gamma generated list is ordered) this may not affect
# the important pyrate stuff anyway, but might affect gen_thumbs.py.
# Going to assume base_ifg_paths is ordered correcly
# pylint: disable=too-many-branches
if params[cf.PROCESSOR] == 2: # if geotif
log.warning("'conv2tif' step not required for geotiff!")
return
mpi_vs_multiprocess_logging("conv2tif", params)
base_ifg_paths = params[cf.INTERFEROGRAM_FILES]
if params[cf.COH_FILE_LIST] is not None:
base_ifg_paths.extend(params[cf.COHERENCE_FILE_PATHS])
if params[cf.DEM_FILE] is not None: # optional DEM conversion
base_ifg_paths.append(params[cf.DEM_FILE_PATH])
process_base_ifgs_paths = np.array_split(base_ifg_paths, mpiops.size)[mpiops.rank]
gtiff_paths = do_geotiff(process_base_ifgs_paths, params)
mpiops.comm.barrier()
log.info("Finished 'conv2tif' step")
return gtiff_paths
def spatio_temporal_filter(params: dict) -> None:
"""
Applies a spatio-temporal filter to remove the atmospheric phase screen
(APS) and saves the corrected interferograms. Firstly the incremental
time series is computed using the SVD method, before a cascade of temporal
then spatial Gaussian filters is applied. The resulting APS corrections are
saved to disc before being subtracted from each interferogram.
:param params: Dictionary of PyRate configuration parameters.
"""
if params[C.APSEST]:
log.info('Doing APS spatio-temporal filtering')
else:
log.info('APS spatio-temporal filtering not required')
return
tiles = params[C.TILES]
preread_ifgs = params[C.PREREAD_IFGS]
ifg_paths = [
ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]
]
# perform some checks on existing ifgs
log.debug('Checking APS correction status')
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_APS_ERROR):
log.debug('Finished APS correction')
return # return if True condition returned
aps_paths = [MultiplePaths.aps_error_path(i, params) for i in ifg_paths]
if all(a.exists() for a in aps_paths):
log.warning('Reusing APS errors from previous run')
_apply_aps_correction(ifg_paths, aps_paths, params)
return
# obtain the incremental time series using SVD
tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles)
mpiops.comm.barrier()
# get lists of epochs and ifgs
ifgs = list(OrderedDict(sorted(preread_ifgs.items())).values())
epochlist = mpiops.run_once(get_epochs, ifgs)[0]
# first perform temporal high pass filter
ts_hp = temporal_high_pass_filter(tsincr, epochlist, params)
# second perform spatial low pass filter to obtain APS correction in ts domain
ifg = Ifg(ifg_paths[0]) # just grab any for parameters in slpfilter
ifg.open()
ts_aps = spatial_low_pass_filter(ts_hp, ifg, params)
ifg.close()
# construct APS corrections for each ifg
_make_aps_corrections(ts_aps, ifgs, params)
# apply correction to ifgs and save ifgs to disc.
_apply_aps_correction(ifg_paths, aps_paths, params)
# update/save the phase_data in the tiled numpy files
shared.save_numpy_phase(ifg_paths, params)
def __reuse_ref_pixel_file_if_exists():
if ref_pixel_file.exists():
refx, refy = np.load(ref_pixel_file)
log.info('Reusing pre-calculated ref-pixel values: ({}, {}) from file {}'.format(
refx, refy, ref_pixel_file.as_posix()))
log.warning("Reusing ref-pixel values from previous run!!!")
params[C.REFX_FOUND], params[C.REFY_FOUND] = int(refx), int(refy)
return int(refx), int(refy)
else:
return None, None
def _save_merged_files(ifgs_dict,
outdir,
array,
out_type,
index=None,
savenpy=None):
"""
Convenience function to save PyRate geotiff and numpy array files
"""
log.debug('Saving PyRate outputs {}'.format(out_type))
gt, md, wkt = ifgs_dict['gt'], ifgs_dict['md'], ifgs_dict['wkt']
epochlist = ifgs_dict['epochlist']
if out_type in ('tsincr', 'tscuml'):
epoch = epochlist.dates[index + 1]
dest = join(outdir, out_type + "_" + str(epoch) + ".tif")
npy_file = join(outdir, out_type + "_" + str(epoch) + ".npy")
# sequence position; first time slice is #0
md['SEQUENCE_POSITION'] = index + 1
md[ifc.EPOCH_DATE] = epoch
else:
dest = join(outdir, out_type + ".tif")
npy_file = join(outdir, out_type + '.npy')
md[ifc.EPOCH_DATE] = [d.strftime('%Y-%m-%d') for d in epochlist.dates]
if out_type == 'stack_rate':
md[ifc.DATA_TYPE] = ifc.STACKRATE
elif out_type == 'stack_error':
md[ifc.DATA_TYPE] = ifc.STACKERROR
elif out_type == 'stack_samples':
md[ifc.DATA_TYPE] = ifc.STACKSAMP
elif out_type == 'linear_rate':
md[ifc.DATA_TYPE] = ifc.LINRATE
elif out_type == 'linear_error':
md[ifc.DATA_TYPE] = ifc.LINERROR
elif out_type == 'linear_samples':
md[ifc.DATA_TYPE] = ifc.LINSAMP
elif out_type == 'linear_intercept':
md[ifc.DATA_TYPE] = ifc.LINICPT
elif out_type == 'linear_rsquared':
md[ifc.DATA_TYPE] = ifc.LINRSQ
elif out_type == 'tsincr':
md[ifc.DATA_TYPE] = ifc.INCR
elif out_type == 'tscuml':
md[ifc.DATA_TYPE] = ifc.CUML
else:
log.warning('Output type "{}" not recognised'.format(out_type))
shared.write_output_geotiff(md, gt, wkt, array, dest, np.nan)
if savenpy:
np.save(file=npy_file, arr=array)
log.debug('Finished saving {}'.format(out_type))
def mpi_vs_multiprocess_logging(step, params):
if mpiops.size > 1: # Over-ride input options if this is an MPI job
log.info(
f"Running '{step}' step with MPI using {mpiops.size} processes")
log.warning(
"Disabling joblib parallel processing (setting parallel = 0)")
params[C.PARALLEL] = 0
else:
if params[C.PARALLEL] == 1:
log.info(
f"Running '{step}' step in parallel using {params[C.PROCESSES]} processes"
)
else:
log.info(f"Running '{step}' step in serial")
def prepare_ifgs(raster_data_paths,
crop_opt,
xlooks,
ylooks,
headers,
params,
thresh=0.5,
user_exts=None,
write_to_disc=True):
"""
Wrapper function to prepare a sequence of interferogram files for
PyRate analysis. See prepifg.prepare_ifg() for full description of
inputs and returns.
Note: function need refining for crop options
:param list raster_data_paths: List of interferogram file paths
:param int crop_opt: Crop option
:param int xlooks: Number of multi-looks in x; 5 is 5 times smaller, 1 is no change
:param int ylooks: Number of multi-looks in y
:param float thresh: see thresh in prepare_ifgs()
:param tuple user_exts: Tuple of user defined georeferenced extents for
new file: (xfirst, yfirst, xlast, ylast)cropping coordinates
:param bool write_to_disc: Write new data to disk
:return: resampled_data: output cropped and resampled image
:rtype: ndarray
:return: out_ds: destination gdal dataset object
:rtype: List[gdal.Dataset]
"""
if xlooks != ylooks:
log.warning('X and Y multi-look factors are not equal')
# use metadata check to check whether it's a dem or ifg
rasters = [dem_or_ifg(r) for r in raster_data_paths]
exts = get_analysis_extent(crop_opt, rasters, xlooks, ylooks, user_exts)
out_paths = []
for r, t in zip(raster_data_paths, rasters):
if isinstance(t, DEM):
input_type = InputTypes.DEM
else:
input_type = InputTypes.IFG
out_path = MultiplePaths(r, params, input_type).sampled_path
out_paths.append(out_path)
return [
prepare_ifg(d, xlooks, ylooks, exts, thresh, crop_opt, h,
write_to_disc, p)
for d, h, p in zip(raster_data_paths, headers, out_paths)
]
def __write_geometry_files(params: dict, exts: Tuple[float, float, float,
float], transform,
ifg_path: str) -> None:
"""
Calculate geometry and save to geotiff files using the information in the
first interferogram in the stack, i.e.:
- rdc_azimuth.tif (azimuth radar coordinate at each pixel)
- rdc_range.tif (range radar coordinate at each pixel)
- azimuth_angle.tif (satellite azimuth angle at each pixel)
- incidence_angle.tif (incidence angle at each pixel)
- look_angle.tif (look angle at each pixel)
"""
ifg = Ifg(ifg_path)
ifg.open(readonly=True)
# calculate per-pixel lon/lat
lon, lat = geometry.get_lonlat_coords(ifg)
# not currently implemented for ROIPAC data which breaks some tests
# if statement can be deleted once ROIPAC is deprecated from PyRate
if ifg.meta_data[ifc.PYRATE_INSAR_PROCESSOR] == 'ROIPAC':
log.warning("Geometry calculations are not implemented for ROI_PAC")
return
# get geometry information and save radar coordinates and angles to tif files
# using metadata of the first image in the stack
# get pixel values of crop (needed to crop lookup table file)
# pixel extent of cropped area (original IFG input)
xmin, ymax = convert_geographic_coordinate_to_pixel_value(
exts[0], exts[1], transform)
xmax, ymin = convert_geographic_coordinate_to_pixel_value(
exts[2], exts[3], transform)
# xmin, xmax: columns of crop
# ymin, ymax: rows of crop
# calculate per-pixel radar coordinates
az, rg = geometry.calc_radar_coords(ifg, params, xmin, xmax, ymin, ymax)
# Read height data from DEM
dem_file = params[C.DEM_FILE_PATH].sampled_path
dem = DEM(dem_file)
# calculate per-pixel look angle (also calculates and saves incidence and azimuth angles)
lk_ang, inc_ang, az_ang, rg_dist = geometry.calc_pixel_geometry(
ifg, rg, lon.data, lat.data, dem.data)
# save radar coordinates and angles to geotiff files
combinations = zip([az, rg, lk_ang, inc_ang, az_ang, rg_dist],
C.GEOMETRY_OUTPUT_TYPES)
shared.iterable_split(__parallelly_write, combinations, params, ifg_path)
def _check_looks(xlooks, ylooks):
"""
Convenience function to verify that looks parameters are valid.
"""
if not (isinstance(xlooks, Number)
and isinstance(ylooks, Number)): # pragma: no cover
msg = "Non-numeric looks parameter(s), x: %s, y: %s" % (xlooks, ylooks)
raise PreprocessError(msg)
if not (xlooks > 0 and ylooks > 0): # pragma: no cover
msg = "Invalid looks parameter(s), x: %s, y: %s. " \
"Looks must be an integer greater than zero" % (xlooks, ylooks)
raise PreprocessError(msg)
if xlooks != ylooks:
log.warning('X and Y multi-look factors are not equal')
def phase_closure_wrapper(params: dict, config: Configuration) -> dict:
"""
This wrapper will run the iterative phase closure check to return a stable
list of checked interferograms, and then mask pixels in interferograms that
exceed the unwrapping error threshold.
:param params: Dictionary of PyRate configuration parameters.
:param config: Configuration class instance.
:return: params: Updated dictionary of PyRate configuration parameters.
"""
if not params[C.PHASE_CLOSURE]:
log.info("Phase closure correction is not required!")
return
rets = iterative_closure_check(config)
if rets is None:
log.info("Zero loops are returned from the iterative closure check.")
log.warning("Abandoning phase closure correction without modifying the interferograms.")
return
ifg_files, ifgs_breach_count, num_occurences_each_ifg = rets
# update params with closure checked ifg list
params[C.INTERFEROGRAM_FILES] = \
mpiops.run_once(update_ifg_list, ifg_files, params[C.INTERFEROGRAM_FILES])
if mpiops.rank == 0:
with open(config.phase_closure_filtered_ifgs_list(params), 'w') as f:
lines = [p.converted_path + '\n' for p in params[C.INTERFEROGRAM_FILES]]
f.writelines(lines)
# mask ifgs with nans where phase unwrap threshold is breached
if mpiops.rank == 0:
mask_pixels_with_unwrapping_errors(ifgs_breach_count, num_occurences_each_ifg, params)
_create_ifg_dict(params) # update the preread_ifgs dict
ifg_paths = [ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]]
# update/save the phase_data in the tiled numpy files
save_numpy_phase(ifg_paths, params)
return params
def remove_orbital_error(ifgs: List, params: dict) -> None:
"""
Wrapper function for PyRate orbital error removal functionality.
NB: the ifg data is modified in situ, rather than create intermediate
files. The network method assumes the given ifgs have already been reduced
to a minimum spanning tree network.
"""
mpiops.run_once(__orb_params_check, params)
ifg_paths = [i.data_path
for i in ifgs] if isinstance(ifgs[0], Ifg) else ifgs
method = params[cf.ORBITAL_FIT_METHOD]
# mlooking is not necessary for independent correction in a computational sense
# can use multiple procesing if write_to_disc=True
if method == INDEPENDENT_METHOD:
log.info('Calculating orbital correction using independent method')
#TODO: implement multi-looking for independent orbit method
if params[cf.ORBITAL_FIT_LOOKS_X] > 1 or params[
cf.ORBITAL_FIT_LOOKS_Y] > 1:
log.warning(
'Multi-looking is not applied in independent orbit method')
ifgs = [shared.Ifg(p)
for p in ifg_paths] if isinstance(ifgs[0], str) else ifgs
process_ifgs = mpiops.array_split(ifgs)
for ifg in process_ifgs:
independent_orbital_correction(ifg, params=params)
elif method == NETWORK_METHOD:
log.info('Calculating orbital correction using network method')
# Here we do all the multilooking in one process, but in memory
# can use multiple processes if we write data to disc during
# remove_orbital_error step
# A performance comparison should be made for saving multilooked
# files on disc vs in memory single process multilooking
if mpiops.rank == MAIN_PROCESS:
mlooked = __create_multilooked_dataset_for_network_correction(
params)
_validate_mlooked(mlooked, ifg_paths)
network_orbital_correction(ifg_paths, params, mlooked)
else:
raise OrbitalError("Unrecognised orbital correction method")
def wrap_spatio_temporal_filter(params):
"""
A wrapper for the spatio-temporal filter so it can be tested.
See docstring for spatio_temporal_filter.
"""
if params[cf.APSEST]:
log.info('Doing APS spatio-temporal filtering')
else:
log.info('APS spatio-temporal filtering not required')
return
tiles = params[cf.TILES]
preread_ifgs = params[cf.PREREAD_IFGS]
ifg_paths = [
ifg_path.tmp_sampled_path
for ifg_path in params[cf.INTERFEROGRAM_FILES]
]
# perform some checks on existing ifgs
log.debug('Checking APS correction status')
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_APS_ERROR):
log.debug('Finished APS correction')
return # return if True condition returned
aps_error_files_on_disc = [
MultiplePaths.aps_error_path(i, params) for i in ifg_paths
]
if all(a.exists() for a in aps_error_files_on_disc):
log.warning("Reusing APS errors from previous run!!!")
for ifg_path, a in mpiops.array_split(
list(zip(ifg_paths, aps_error_files_on_disc))):
phase = np.load(a)
_save_aps_corrected_phase(ifg_path, phase)
else:
tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles)
mpiops.comm.barrier()
spatio_temporal_filter(tsincr, ifg_paths, params, preread_ifgs)
mpiops.comm.barrier()
shared.save_numpy_phase(ifg_paths, params)
def main(params: dict) -> None:
"""
PyRate merge main function. Assembles product tiles in to
single geotiff files
"""
if params[C.SIGNAL_POLARITY] == 1:
log.info(
f"Saving output products with the same sign convention as input data"
)
elif params[C.SIGNAL_POLARITY] == -1:
log.info(
f"Saving output products with reversed sign convention compared to the input data"
)
else:
log.warning(f"Check the value of signal_polarity parameter")
out_types = []
tsfile = join(params[C.TMPDIR], 'tscuml_0.npy')
if exists(tsfile):
_merge_timeseries(params, 'tscuml')
_merge_linrate(params)
out_types += ['linear_rate', 'linear_error', 'linear_rsquared']
# optional save of merged tsincr products
if params["savetsincr"] == 1:
_merge_timeseries(params, 'tsincr')
else:
log.warning(
'Not merging time series products; {} does not exist'.format(
tsfile))
stfile = join(params[C.TMPDIR], 'stack_rate_0.npy')
if exists(stfile):
# setup paths
mpiops.run_once(_merge_stack, params)
out_types += ['stack_rate', 'stack_error']
else:
log.warning(
'Not merging stack products; {} does not exist'.format(stfile))
if len(out_types) > 0:
process_out_types = mpiops.array_split(out_types)
for out_type in process_out_types:
create_png_and_kml_from_tif(params[C.VELOCITY_DIR],
output_type=out_type)
else:
log.warning('Exiting: no products to merge')
def dem_error_calc_wrapper(params: dict) -> None:
"""
MPI wrapper for DEM error correction
:param params: Dictionary of PyRate configuration parameters.
"""
if not params[C.DEMERROR]:
log.info("DEM error correction not required")
return
# geometry information needed to calculate Bperp for each pixel using first IFG in list
ifg_paths = [
ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]
]
# check if DEM error correction is already available
if mpiops.run_once(__check_and_apply_demerrors_found_on_disc, ifg_paths,
params):
log.warning("Reusing DEM error correction from previous run!!!")
else:
log.info("Calculating DEM error correction")
# read and open the first IFG in list
ifg0_path = ifg_paths[0]
ifg0 = Ifg(ifg0_path)
ifg0.open(readonly=True)
# not currently implemented for ROIPAC data which breaks some tests
# if statement can be deleted once ROIPAC is deprecated from PyRate
if ifg0.meta_data[ifc.PYRATE_INSAR_PROCESSOR] == 'ROIPAC':
log.warning(
"Geometry calculations are not implemented for ROI_PAC")
return
if params[C.BASE_FILE_LIST] is None:
log.warning(
"No baseline files supplied: DEM error has not been computed")
return
# todo: subtract other corrections (e.g. orbital) from displacement phase before estimating the DEM error
# the following code is a quick way to do the bperp calculation, but is not identical to the GAMMA output
# where the near range of the first SLC is used for each pair.
# calculate look angle for interferograms (using the Near Range of the first SLC)
# look_angle = geometry.calc_local_geometry(ifg0, None, rg, lon, lat, params)
# bperp = geometry.calc_local_baseline(ifg0, az, look_angle)
# split full arrays in to tiles for parallel processing
tiles_split(_process_dem_error_per_tile, params)
preread_ifgs = params[C.PREREAD_IFGS]
# write dem error and correction values to file
mpiops.run_once(_write_dem_errors, ifg_paths, params, preread_ifgs)
shared.save_numpy_phase(ifg_paths, params)
log.debug('Finished DEM error correction step')
def main(params: dict) -> None:
"""
PyRate merge main function. Assembles product tiles in to
single geotiff files
"""
out_types = []
stfile = join(params[cf.TMPDIR], 'stack_rate_0.npy')
if exists(stfile):
# setup paths
mpiops.run_once(_merge_stack, params)
out_types += ['stack_rate', 'stack_error']
else:
log.warning(
'Not merging stack products; {} does not exist'.format(stfile))
tsfile = join(params[cf.TMPDIR], 'tscuml_0.npy')
if exists(tsfile):
_merge_timeseries(params, 'tscuml')
_merge_linrate(params)
out_types += ['linear_rate', 'linear_error', 'linear_rsquared']
# optional save of merged tsincr products
if params["savetsincr"] == 1:
_merge_timeseries(params, 'tsincr')
else:
log.warning(
'Not merging time series products; {} does not exist'.format(
tsfile))
if len(out_types) > 0:
process_out_types = mpiops.array_split(out_types)
for out_type in process_out_types:
create_png_and_kml_from_tif(params[cf.OUT_DIR],
output_type=out_type)
else:
log.warning('Exiting: no products to merge')
def network_orbital_correction(ifg_paths,
params,
m_ifgs: Optional[List] = None):
"""
This algorithm implements a network inversion to determine orbital
corrections for a set of interferograms forming a connected network.
Warning: This will write orbital error corrected phase_data to the ifgs.
:param list ifg_paths: List of Ifg class objects reduced to a minimum spanning
tree network
:param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC)
:param bool offset: True to calculate the model using offsets
:param dict params: dictionary of configuration parameters
:param list m_ifgs: list of multilooked Ifg class objects
(sequence must be multilooked versions of 'ifgs' arg)
:param dict preread_ifgs: Dictionary containing information specifically
for MPI jobs (optional)
:return: None - interferogram phase data is updated and saved to disk
"""
# pylint: disable=too-many-locals, too-many-arguments
offset = params[cf.ORBFIT_OFFSET]
degree = params[cf.ORBITAL_FIT_DEGREE]
preread_ifgs = params[cf.PREREAD_IFGS]
# all orbit corrections available?
if isinstance(ifg_paths[0], str):
if __check_and_apply_orberrors_found_on_disc(ifg_paths, params):
log.warning("Reusing orbfit errors from previous run!!!")
return
# all corrections are available in numpy files already saved - return
ifgs = [shared.Ifg(i) for i in ifg_paths]
else: # alternate test paths # TODO: improve
ifgs = ifg_paths
src_ifgs = ifgs if m_ifgs is None else m_ifgs
src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst
vphase = vstack([i.phase_data.reshape((i.num_cells, 1)) for i in src_ifgs])
vphase = squeeze(vphase)
B = get_network_design_matrix(src_ifgs, degree, offset)
# filter NaNs out before getting model
B = B[~isnan(vphase)]
orbparams = dot(pinv(B, 1e-6), vphase[~isnan(vphase)])
ncoef = _get_num_params(degree)
if preread_ifgs:
temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values()
ids = first_second_ids(get_all_epochs(temp_ifgs))
else:
ids = first_second_ids(get_all_epochs(ifgs))
coefs = [
orbparams[i:i + ncoef] for i in range(0,
len(set(ids)) * ncoef, ncoef)
]
# create full res DM to expand determined coefficients into full res
# orbital correction (eg. expand coarser model to full size)
if preread_ifgs:
temp_ifg = Ifg(ifg_paths[0]) # ifgs here are paths
temp_ifg.open()
dm = get_design_matrix(temp_ifg, degree, offset=False)
temp_ifg.close()
else:
ifg = ifgs[0]
dm = get_design_matrix(ifg, degree, offset=False)
for i in ifg_paths:
# open if not Ifg instance
if isinstance(i, str): # pragma: no cover
# are paths
i = Ifg(i)
i.open(readonly=False)
shared.nan_and_mm_convert(i, params)
_remove_network_orb_error(coefs, dm, i, ids, offset, params)
def _ref_pixel_calc(ifg_paths: List[str], params: dict) -> Tuple[int, int]:
"""
Wrapper for reference pixel calculation
"""
lon = params[cf.REFX]
lat = params[cf.REFY]
ifg = Ifg(ifg_paths[0])
ifg.open(readonly=True)
# assume all interferograms have same projection and will share the same transform
transform = ifg.dataset.GetGeoTransform()
if lon == -1 or lat == -1:
log.info('Searching for best reference pixel location')
half_patch_size, thresh, grid = refpixel.ref_pixel_setup(
ifg_paths, params)
process_grid = mpiops.array_split(grid)
refpixel.save_ref_pixel_blocks(process_grid, half_patch_size,
ifg_paths, params)
mean_sds = refpixel._ref_pixel_mpi(process_grid, half_patch_size,
ifg_paths, thresh, params)
mean_sds = mpiops.comm.gather(mean_sds, root=0)
if mpiops.rank == MASTER_PROCESS:
mean_sds = np.hstack(mean_sds)
refpixel_returned = mpiops.run_once(refpixel.find_min_mean, mean_sds,
grid)
if isinstance(refpixel_returned, ValueError):
from pyrate.core.refpixel import RefPixelError
raise RefPixelError(
"Reference pixel calculation returned an all nan slice!\n"
"Cannot continue downstream computation. Please change reference pixel algorithm used before "
"continuing.")
refy, refx = refpixel_returned # row first means first value is latitude
log.info('Selected reference pixel coordinate (x, y): ({}, {})'.format(
refx, refy))
lon, lat = refpixel.convert_pixel_value_to_geographic_coordinate(
refx, refy, transform)
log.info(
'Selected reference pixel coordinate (lon, lat): ({}, {})'.format(
lon, lat))
else:
log.info('Using reference pixel from config file (lon, lat): ({}, {})'.
format(lon, lat))
log.warning(
"Ensure user supplied reference pixel values are in lon/lat")
refx, refy = refpixel.convert_geographic_coordinate_to_pixel_value(
lon, lat, transform)
log.info(
'Converted reference pixel coordinate (x, y): ({}, {})'.format(
refx, refy))
refpixel.update_refpix_metadata(ifg_paths, refx, refy, transform, params)
log.debug("refpx, refpy: " + str(refx) + " " + str(refy))
ifg.close()
return int(refx), int(refy)
def main():
parser = argparse.ArgumentParser(
prog='plot_ifgs',
description="Python script to plot interferograms",
add_help=True,
formatter_class=RawTextHelpFormatter)
parser.add_argument('-v',
'--verbosity',
type=str,
default='INFO',
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'],
help="Increase output verbosity")
parser.add_argument('-d',
'--directory',
action="store",
type=str,
default=None,
help="Pass path to directory containing ifgs",
required=True)
parser.add_argument('-f',
'--config_file',
action="store",
type=str,
default=None,
help="Pass configuration file",
required=True)
parser.add_argument('-n',
'--ifgs_per_plot',
type=int,
default=50,
help='number of ifgs per plot',
required=False)
args = parser.parse_args()
params = _params_from_conf(args.config_file)
configure_stage_log(
args.verbosity, 'plot_ifgs',
Path(params[C.OUT_DIR]).joinpath('pyrate.log.').as_posix())
if args.verbosity:
log.setLevel(args.verbosity)
log.info("Verbosity set to " + str(args.verbosity) + ".")
log.debug("Arguments supplied at command line: ")
log.debug(args)
ifgs = sorted(list(Path(args.directory).glob('*_ifg.tif')))
num_ifgs = len(ifgs)
if num_ifgs == 0:
log.warning(
f'No interferograms with extension *_ifg.tif were found in {args.directory}'
)
return
log.info(f'Plotting {num_ifgs} interferograms found in {args.directory}')
ifgs_per_plot = args.ifgs_per_plot
plt_rows = np.int(np.sqrt(ifgs_per_plot))
plt_cols = ifgs_per_plot // plt_rows
if ifgs_per_plot % plt_rows:
plt_cols += 1
tot_plots = 0
num_of_figs = math.ceil(num_ifgs / ifgs_per_plot)
fig_no = 0
for i in range(num_of_figs):
fig_no += 1
fig = plt.figure(figsize=(12 * plt_rows, 8 * plt_cols))
fig_plots = 0
for p_r in range(plt_rows):
for p_c in range(plt_cols):
ax = fig.add_subplot(plt_rows, plt_cols, fig_plots + 1)
ifg_num = plt_cols * p_r + p_c + ifgs_per_plot * i
file = ifgs[ifg_num]
log.info(f'Plotting {file}')
__plot_ifg(file, cmap, ax, num_ifgs, params)
tot_plots += 1
fig_plots += 1
log.debug(f'Plotted interferogram #{tot_plots}')
if (fig_plots == ifgs_per_plot) or (tot_plots == num_ifgs):
f_name = Path(args.directory).joinpath('ifg-phase-plot-' +
str(fig_no) +
'.png').as_posix()
plt.savefig(f_name, dpi=50)
log.info(f'{fig_plots} interferograms plotted in {f_name}')
plt.close(fig)
break
if tot_plots == num_ifgs:
break
def ref_pixel_calc_wrapper(params: dict) -> Tuple[int, int]:
"""
Wrapper for reference pixel calculation
"""
__validate_supplied_lat_lon(params)
ifg_paths = [ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]]
lon = params[C.REFX]
lat = params[C.REFY]
ifg = Ifg(ifg_paths[0])
ifg.open(readonly=True)
# assume all interferograms have same projection and will share the same transform
transform = ifg.dataset.GetGeoTransform()
ref_pixel_file = Configuration.ref_pixel_path(params)
def __reuse_ref_pixel_file_if_exists():
if ref_pixel_file.exists():
refx, refy = np.load(ref_pixel_file)
log.info('Reusing pre-calculated ref-pixel values: ({}, {}) from file {}'.format(
refx, refy, ref_pixel_file.as_posix()))
log.warning("Reusing ref-pixel values from previous run!!!")
params[C.REFX_FOUND], params[C.REFY_FOUND] = int(refx), int(refy)
return int(refx), int(refy)
else:
return None, None
# read and return
refx, refy = mpiops.run_once(__reuse_ref_pixel_file_if_exists)
if (refx is not None) and (refy is not None):
update_refpix_metadata(ifg_paths, int(refx), int(refy), transform, params)
return refx, refy
if lon == -1 or lat == -1:
log.info('Searching for best reference pixel location')
half_patch_size, thresh, grid = ref_pixel_setup(ifg_paths, params)
process_grid = mpiops.array_split(grid)
save_ref_pixel_blocks(process_grid, half_patch_size, ifg_paths, params)
mean_sds = _ref_pixel_mpi(process_grid, half_patch_size, ifg_paths, thresh, params)
mean_sds = mpiops.comm.gather(mean_sds, root=0)
if mpiops.rank == MAIN_PROCESS:
mean_sds = np.hstack(mean_sds)
refpixel_returned = mpiops.run_once(find_min_mean, mean_sds, grid)
if isinstance(refpixel_returned, ValueError):
raise RefPixelError(
"Reference pixel calculation returned an all nan slice!\n"
"Cannot continue downstream computation. Please change reference pixel algorithm used before "
"continuing.")
refy, refx = refpixel_returned # row first means first value is latitude
log.info('Selected reference pixel coordinate (x, y): ({}, {})'.format(refx, refy))
lon, lat = convert_pixel_value_to_geographic_coordinate(refx, refy, transform)
log.info('Selected reference pixel coordinate (lon, lat): ({}, {})'.format(lon, lat))
else:
log.info('Using reference pixel from config file (lon, lat): ({}, {})'.format(lon, lat))
log.warning("Ensure user supplied reference pixel values are in lon/lat")
refx, refy = convert_geographic_coordinate_to_pixel_value(lon, lat, transform)
log.info('Converted reference pixel coordinate (x, y): ({}, {})'.format(refx, refy))
np.save(file=ref_pixel_file, arr=[int(refx), int(refy)])
update_refpix_metadata(ifg_paths, refx, refy, transform, params)
log.debug("refpx, refpy: "+str(refx) + " " + str(refy))
ifg.close()
params[C.REFX_FOUND], params[C.REFY_FOUND] = int(refx), int(refy)
return int(refx), int(refy)
def ref_phase_est_wrapper(params):
"""
Wrapper for reference phase estimation.
"""
ifg_paths = [
ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]
]
refpx, refpy = params[C.REFX_FOUND], params[C.REFY_FOUND]
if len(ifg_paths) < 2:
raise ReferencePhaseError(
"At least two interferograms required for reference phase correction ({len_ifg_paths} "
"provided).".format(len_ifg_paths=len(ifg_paths)))
# this is not going to be true as we now start with fresh multilooked ifg copies - remove?
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_REF_PHASE):
log.warning(
'Reference phase correction already applied to ifgs; returning')
return
ifgs = [Ifg(ifg_path) for ifg_path in ifg_paths]
# Save reference phase numpy arrays to disk.
ref_phs_file = Configuration.ref_phs_file(params)
# If ref phase file exists on disk, then reuse - subtract ref_phase from ifgs and return
if ref_phs_file.exists():
ref_phs = np.load(ref_phs_file)
_update_phase_and_metadata(ifgs, ref_phs, params)
shared.save_numpy_phase(ifg_paths, params)
return ref_phs, ifgs
# determine the reference phase for each ifg
if params[C.REF_EST_METHOD] == 1:
log.info("Calculating reference phase as median of interferogram")
ref_phs = est_ref_phase_ifg_median(ifg_paths, params)
elif params[C.REF_EST_METHOD] == 2:
log.info(
'Calculating reference phase in a patch surrounding pixel (x, y): ({}, {})'
.format(refpx, refpy))
ref_phs = est_ref_phase_patch_median(ifg_paths, params, refpx, refpy)
else:
raise ReferencePhaseError(
"No such option, set parameter 'refest' to '1' or '2'.")
# gather all reference phases from distributed processes and save to disk
if mpiops.rank == MAIN_PROCESS:
collected_ref_phs = np.zeros(len(ifg_paths), dtype=np.float64)
process_indices = mpiops.array_split(range(len(ifg_paths))).astype(
np.uint16)
collected_ref_phs[process_indices] = ref_phs
for r in range(1, mpiops.size):
process_indices = mpiops.array_split(range(len(ifg_paths)),
r).astype(np.uint16)
this_process_ref_phs = np.zeros(shape=len(process_indices),
dtype=np.float64)
mpiops.comm.Recv(this_process_ref_phs, source=r, tag=r)
collected_ref_phs[process_indices] = this_process_ref_phs
np.save(file=ref_phs_file, arr=collected_ref_phs)
else:
collected_ref_phs = np.empty(len(ifg_paths), dtype=np.float64)
mpiops.comm.Send(ref_phs, dest=MAIN_PROCESS, tag=mpiops.rank)
mpiops.comm.Bcast(collected_ref_phs, root=0)
# subtract ref_phase from ifgs
_update_phase_and_metadata(ifgs, collected_ref_phs, params)
mpiops.comm.barrier()
shared.save_numpy_phase(ifg_paths, params)
log.debug("Finished reference phase correction")
# Preserve old return value so tests don't break.
return ref_phs, ifgs
def network_orbital_correction(ifg_paths,
params,
m_ifgs: Optional[List] = None):
"""
This algorithm implements a network inversion to determine orbital
corrections for a set of interferograms forming a connected network.
Warning: This will write orbital error corrected phase_data to the ifgs.
:param list ifg_paths: List of Ifg class objects reduced to a minimum spanning tree network
:param dict params: dictionary of configuration parameters
:param list m_ifgs: list of multilooked Ifg class objects (sequence must be multilooked versions of 'ifgs' arg)
:return: None - interferogram phase data is updated and saved to disk
"""
# pylint: disable=too-many-locals, too-many-arguments
offset = params[C.ORBFIT_OFFSET]
degree = params[C.ORBITAL_FIT_DEGREE]
preread_ifgs = params[C.PREREAD_IFGS]
intercept = params[C.ORBFIT_INTERCEPT]
scale = params[C.ORBFIT_SCALE]
# all orbit corrections available?
if isinstance(ifg_paths[0], str):
if __check_and_apply_orberrors_found_on_disc(ifg_paths, params):
log.warning("Reusing orbfit errors from previous run!!!")
return
# all corrections are available in numpy files already saved - return
ifgs = [shared.Ifg(i) for i in ifg_paths]
else: # alternate test paths # TODO: improve
ifgs = ifg_paths
src_ifgs = ifgs if m_ifgs is None else m_ifgs
src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst
if preread_ifgs:
temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values()
ids = first_second_ids(get_all_epochs(temp_ifgs))
else:
ids = first_second_ids(get_all_epochs(ifgs))
nepochs = len(set(ids))
# call the actual inversion routine
coefs = calc_network_orb_correction(src_ifgs,
degree,
scale,
nepochs,
intercept=intercept)
# create full res DM to expand determined coefficients into full res
# orbital correction (eg. expand coarser model to full size)
if preread_ifgs:
temp_ifg = Ifg(ifg_paths[0]) # ifgs here are paths
temp_ifg.open()
dm = get_design_matrix(temp_ifg,
degree,
intercept=intercept,
scale=scale)
temp_ifg.close()
else:
ifg = ifgs[0]
dm = get_design_matrix(ifg, degree, intercept=intercept, scale=scale)
for i in ifg_paths:
# open if not Ifg instance
if isinstance(i, str): # pragma: no cover
# are paths
i = Ifg(i)
i.open(readonly=False)
shared.nan_and_mm_convert(i, params)
_remove_network_orb_error(coefs, dm, i, ids, offset, params)
