def _geotiff_multiprocessing(unw_path: MultiplePaths, params: dict) -> Tuple[str, bool]: """ Multiprocessing wrapper for full-res geotiff conversion """ # TODO: Need a more robust method for identifying coherence files. dest = unw_path.converted_path processor = params[C.PROCESSOR] # roipac or gamma # Create full-res geotiff if not already on disk if not os.path.exists(dest): if processor == GAMMA: header = gamma.gamma_header(unw_path.unwrapped_path, params) elif processor == ROIPAC: log.info( "Warning: ROI_PAC support will be deprecated in a future PyRate release" ) header = roipac.roipac_header(unw_path.unwrapped_path, params) else: raise PreprocessError('Processor must be ROI_PAC (0) or GAMMA (1)') header[ifc.INPUT_TYPE] = unw_path.input_type shared.write_fullres_geotiff(header, unw_path.unwrapped_path, dest, nodata=params[C.NO_DATA_VALUE]) Path(dest).chmod(0o444) # readonly output return dest, True else: log.warning( f"Full-res geotiff already exists in {dest}! Returning existing geotiff!" ) return dest, False
def convert_to_nans(self): """ Convert phase data of given value to NaN """ if (self._nodata_value is None) \ or (self.dataset is None): # pragma: no cover msg = 'nodata value needs to be set for nan conversion.' \ 'Use ifg.nodata_value = NoDataValue to set nodata_value' log.warning(msg) raise RasterException(msg) if ((self.dataset.GetMetadataItem(ifc.NAN_STATUS) == ifc.NAN_CONVERTED) or self.nan_converted): self.phase_data = self.phase_data self.nan_converted = True msg = '{}: ignored as previous nan ' \ 'conversion detected'.format(self.data_path) log.debug(msg) return else: self.phase_data = where( isclose(self.phase_data, self._nodata_value, atol=1e-6), nan, self.phase_data) self.meta_data[ifc.NAN_STATUS] = ifc.NAN_CONVERTED self.nan_converted = True
def main(params): """ Parse parameters and prepare files for conversion. :param dict params: Parameters dictionary read in from the config file """ # TODO: looks like base_ifg_paths are ordered according to ifg list # This probably won't be a problem because input list won't be reordered # and the original gamma generated list is ordered) this may not affect # the important pyrate stuff anyway, but might affect gen_thumbs.py. # Going to assume base_ifg_paths is ordered correcly # pylint: disable=too-many-branches if params[cf.PROCESSOR] == 2: # if geotif log.warning("'conv2tif' step not required for geotiff!") return mpi_vs_multiprocess_logging("conv2tif", params) base_ifg_paths = params[cf.INTERFEROGRAM_FILES] if params[cf.COH_FILE_LIST] is not None: base_ifg_paths.extend(params[cf.COHERENCE_FILE_PATHS]) if params[cf.DEM_FILE] is not None: # optional DEM conversion base_ifg_paths.append(params[cf.DEM_FILE_PATH]) process_base_ifgs_paths = np.array_split(base_ifg_paths, mpiops.size)[mpiops.rank] gtiff_paths = do_geotiff(process_base_ifgs_paths, params) mpiops.comm.barrier() log.info("Finished 'conv2tif' step") return gtiff_paths
def spatio_temporal_filter(params: dict) -> None: """ Applies a spatio-temporal filter to remove the atmospheric phase screen (APS) and saves the corrected interferograms. Firstly the incremental time series is computed using the SVD method, before a cascade of temporal then spatial Gaussian filters is applied. The resulting APS corrections are saved to disc before being subtracted from each interferogram. :param params: Dictionary of PyRate configuration parameters. """ if params[C.APSEST]: log.info('Doing APS spatio-temporal filtering') else: log.info('APS spatio-temporal filtering not required') return tiles = params[C.TILES] preread_ifgs = params[C.PREREAD_IFGS] ifg_paths = [ ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES] ] # perform some checks on existing ifgs log.debug('Checking APS correction status') if mpiops.run_once(shared.check_correction_status, ifg_paths, ifc.PYRATE_APS_ERROR): log.debug('Finished APS correction') return # return if True condition returned aps_paths = [MultiplePaths.aps_error_path(i, params) for i in ifg_paths] if all(a.exists() for a in aps_paths): log.warning('Reusing APS errors from previous run') _apply_aps_correction(ifg_paths, aps_paths, params) return # obtain the incremental time series using SVD tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles) mpiops.comm.barrier() # get lists of epochs and ifgs ifgs = list(OrderedDict(sorted(preread_ifgs.items())).values()) epochlist = mpiops.run_once(get_epochs, ifgs)[0] # first perform temporal high pass filter ts_hp = temporal_high_pass_filter(tsincr, epochlist, params) # second perform spatial low pass filter to obtain APS correction in ts domain ifg = Ifg(ifg_paths[0]) # just grab any for parameters in slpfilter ifg.open() ts_aps = spatial_low_pass_filter(ts_hp, ifg, params) ifg.close() # construct APS corrections for each ifg _make_aps_corrections(ts_aps, ifgs, params) # apply correction to ifgs and save ifgs to disc. _apply_aps_correction(ifg_paths, aps_paths, params) # update/save the phase_data in the tiled numpy files shared.save_numpy_phase(ifg_paths, params)
def __reuse_ref_pixel_file_if_exists(): if ref_pixel_file.exists(): refx, refy = np.load(ref_pixel_file) log.info('Reusing pre-calculated ref-pixel values: ({}, {}) from file {}'.format( refx, refy, ref_pixel_file.as_posix())) log.warning("Reusing ref-pixel values from previous run!!!") params[C.REFX_FOUND], params[C.REFY_FOUND] = int(refx), int(refy) return int(refx), int(refy) else: return None, None
def _save_merged_files(ifgs_dict, outdir, array, out_type, index=None, savenpy=None): """ Convenience function to save PyRate geotiff and numpy array files """ log.debug('Saving PyRate outputs {}'.format(out_type)) gt, md, wkt = ifgs_dict['gt'], ifgs_dict['md'], ifgs_dict['wkt'] epochlist = ifgs_dict['epochlist'] if out_type in ('tsincr', 'tscuml'): epoch = epochlist.dates[index + 1] dest = join(outdir, out_type + "_" + str(epoch) + ".tif") npy_file = join(outdir, out_type + "_" + str(epoch) + ".npy") # sequence position; first time slice is #0 md['SEQUENCE_POSITION'] = index + 1 md[ifc.EPOCH_DATE] = epoch else: dest = join(outdir, out_type + ".tif") npy_file = join(outdir, out_type + '.npy') md[ifc.EPOCH_DATE] = [d.strftime('%Y-%m-%d') for d in epochlist.dates] if out_type == 'stack_rate': md[ifc.DATA_TYPE] = ifc.STACKRATE elif out_type == 'stack_error': md[ifc.DATA_TYPE] = ifc.STACKERROR elif out_type == 'stack_samples': md[ifc.DATA_TYPE] = ifc.STACKSAMP elif out_type == 'linear_rate': md[ifc.DATA_TYPE] = ifc.LINRATE elif out_type == 'linear_error': md[ifc.DATA_TYPE] = ifc.LINERROR elif out_type == 'linear_samples': md[ifc.DATA_TYPE] = ifc.LINSAMP elif out_type == 'linear_intercept': md[ifc.DATA_TYPE] = ifc.LINICPT elif out_type == 'linear_rsquared': md[ifc.DATA_TYPE] = ifc.LINRSQ elif out_type == 'tsincr': md[ifc.DATA_TYPE] = ifc.INCR elif out_type == 'tscuml': md[ifc.DATA_TYPE] = ifc.CUML else: log.warning('Output type "{}" not recognised'.format(out_type)) shared.write_output_geotiff(md, gt, wkt, array, dest, np.nan) if savenpy: np.save(file=npy_file, arr=array) log.debug('Finished saving {}'.format(out_type))
def mpi_vs_multiprocess_logging(step, params): if mpiops.size > 1: # Over-ride input options if this is an MPI job log.info( f"Running '{step}' step with MPI using {mpiops.size} processes") log.warning( "Disabling joblib parallel processing (setting parallel = 0)") params[C.PARALLEL] = 0 else: if params[C.PARALLEL] == 1: log.info( f"Running '{step}' step in parallel using {params[C.PROCESSES]} processes" ) else: log.info(f"Running '{step}' step in serial")
def prepare_ifgs(raster_data_paths, crop_opt, xlooks, ylooks, headers, params, thresh=0.5, user_exts=None, write_to_disc=True): """ Wrapper function to prepare a sequence of interferogram files for PyRate analysis. See prepifg.prepare_ifg() for full description of inputs and returns. Note: function need refining for crop options :param list raster_data_paths: List of interferogram file paths :param int crop_opt: Crop option :param int xlooks: Number of multi-looks in x; 5 is 5 times smaller, 1 is no change :param int ylooks: Number of multi-looks in y :param float thresh: see thresh in prepare_ifgs() :param tuple user_exts: Tuple of user defined georeferenced extents for new file: (xfirst, yfirst, xlast, ylast)cropping coordinates :param bool write_to_disc: Write new data to disk :return: resampled_data: output cropped and resampled image :rtype: ndarray :return: out_ds: destination gdal dataset object :rtype: List[gdal.Dataset] """ if xlooks != ylooks: log.warning('X and Y multi-look factors are not equal') # use metadata check to check whether it's a dem or ifg rasters = [dem_or_ifg(r) for r in raster_data_paths] exts = get_analysis_extent(crop_opt, rasters, xlooks, ylooks, user_exts) out_paths = [] for r, t in zip(raster_data_paths, rasters): if isinstance(t, DEM): input_type = InputTypes.DEM else: input_type = InputTypes.IFG out_path = MultiplePaths(r, params, input_type).sampled_path out_paths.append(out_path) return [ prepare_ifg(d, xlooks, ylooks, exts, thresh, crop_opt, h, write_to_disc, p) for d, h, p in zip(raster_data_paths, headers, out_paths) ]
def __write_geometry_files(params: dict, exts: Tuple[float, float, float, float], transform, ifg_path: str) -> None: """ Calculate geometry and save to geotiff files using the information in the first interferogram in the stack, i.e.: - rdc_azimuth.tif (azimuth radar coordinate at each pixel) - rdc_range.tif (range radar coordinate at each pixel) - azimuth_angle.tif (satellite azimuth angle at each pixel) - incidence_angle.tif (incidence angle at each pixel) - look_angle.tif (look angle at each pixel) """ ifg = Ifg(ifg_path) ifg.open(readonly=True) # calculate per-pixel lon/lat lon, lat = geometry.get_lonlat_coords(ifg) # not currently implemented for ROIPAC data which breaks some tests # if statement can be deleted once ROIPAC is deprecated from PyRate if ifg.meta_data[ifc.PYRATE_INSAR_PROCESSOR] == 'ROIPAC': log.warning("Geometry calculations are not implemented for ROI_PAC") return # get geometry information and save radar coordinates and angles to tif files # using metadata of the first image in the stack # get pixel values of crop (needed to crop lookup table file) # pixel extent of cropped area (original IFG input) xmin, ymax = convert_geographic_coordinate_to_pixel_value( exts[0], exts[1], transform) xmax, ymin = convert_geographic_coordinate_to_pixel_value( exts[2], exts[3], transform) # xmin, xmax: columns of crop # ymin, ymax: rows of crop # calculate per-pixel radar coordinates az, rg = geometry.calc_radar_coords(ifg, params, xmin, xmax, ymin, ymax) # Read height data from DEM dem_file = params[C.DEM_FILE_PATH].sampled_path dem = DEM(dem_file) # calculate per-pixel look angle (also calculates and saves incidence and azimuth angles) lk_ang, inc_ang, az_ang, rg_dist = geometry.calc_pixel_geometry( ifg, rg, lon.data, lat.data, dem.data) # save radar coordinates and angles to geotiff files combinations = zip([az, rg, lk_ang, inc_ang, az_ang, rg_dist], C.GEOMETRY_OUTPUT_TYPES) shared.iterable_split(__parallelly_write, combinations, params, ifg_path)
def _check_looks(xlooks, ylooks): """ Convenience function to verify that looks parameters are valid. """ if not (isinstance(xlooks, Number) and isinstance(ylooks, Number)): # pragma: no cover msg = "Non-numeric looks parameter(s), x: %s, y: %s" % (xlooks, ylooks) raise PreprocessError(msg) if not (xlooks > 0 and ylooks > 0): # pragma: no cover msg = "Invalid looks parameter(s), x: %s, y: %s. " \ "Looks must be an integer greater than zero" % (xlooks, ylooks) raise PreprocessError(msg) if xlooks != ylooks: log.warning('X and Y multi-look factors are not equal')
def phase_closure_wrapper(params: dict, config: Configuration) -> dict: """ This wrapper will run the iterative phase closure check to return a stable list of checked interferograms, and then mask pixels in interferograms that exceed the unwrapping error threshold. :param params: Dictionary of PyRate configuration parameters. :param config: Configuration class instance. :return: params: Updated dictionary of PyRate configuration parameters. """ if not params[C.PHASE_CLOSURE]: log.info("Phase closure correction is not required!") return rets = iterative_closure_check(config) if rets is None: log.info("Zero loops are returned from the iterative closure check.") log.warning("Abandoning phase closure correction without modifying the interferograms.") return ifg_files, ifgs_breach_count, num_occurences_each_ifg = rets # update params with closure checked ifg list params[C.INTERFEROGRAM_FILES] = \ mpiops.run_once(update_ifg_list, ifg_files, params[C.INTERFEROGRAM_FILES]) if mpiops.rank == 0: with open(config.phase_closure_filtered_ifgs_list(params), 'w') as f: lines = [p.converted_path + '\n' for p in params[C.INTERFEROGRAM_FILES]] f.writelines(lines) # mask ifgs with nans where phase unwrap threshold is breached if mpiops.rank == 0: mask_pixels_with_unwrapping_errors(ifgs_breach_count, num_occurences_each_ifg, params) _create_ifg_dict(params) # update the preread_ifgs dict ifg_paths = [ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]] # update/save the phase_data in the tiled numpy files save_numpy_phase(ifg_paths, params) return params
def remove_orbital_error(ifgs: List, params: dict) -> None: """ Wrapper function for PyRate orbital error removal functionality. NB: the ifg data is modified in situ, rather than create intermediate files. The network method assumes the given ifgs have already been reduced to a minimum spanning tree network. """ mpiops.run_once(__orb_params_check, params) ifg_paths = [i.data_path for i in ifgs] if isinstance(ifgs[0], Ifg) else ifgs method = params[cf.ORBITAL_FIT_METHOD] # mlooking is not necessary for independent correction in a computational sense # can use multiple procesing if write_to_disc=True if method == INDEPENDENT_METHOD: log.info('Calculating orbital correction using independent method') #TODO: implement multi-looking for independent orbit method if params[cf.ORBITAL_FIT_LOOKS_X] > 1 or params[ cf.ORBITAL_FIT_LOOKS_Y] > 1: log.warning( 'Multi-looking is not applied in independent orbit method') ifgs = [shared.Ifg(p) for p in ifg_paths] if isinstance(ifgs[0], str) else ifgs process_ifgs = mpiops.array_split(ifgs) for ifg in process_ifgs: independent_orbital_correction(ifg, params=params) elif method == NETWORK_METHOD: log.info('Calculating orbital correction using network method') # Here we do all the multilooking in one process, but in memory # can use multiple processes if we write data to disc during # remove_orbital_error step # A performance comparison should be made for saving multilooked # files on disc vs in memory single process multilooking if mpiops.rank == MAIN_PROCESS: mlooked = __create_multilooked_dataset_for_network_correction( params) _validate_mlooked(mlooked, ifg_paths) network_orbital_correction(ifg_paths, params, mlooked) else: raise OrbitalError("Unrecognised orbital correction method")
def wrap_spatio_temporal_filter(params): """ A wrapper for the spatio-temporal filter so it can be tested. See docstring for spatio_temporal_filter. """ if params[cf.APSEST]: log.info('Doing APS spatio-temporal filtering') else: log.info('APS spatio-temporal filtering not required') return tiles = params[cf.TILES] preread_ifgs = params[cf.PREREAD_IFGS] ifg_paths = [ ifg_path.tmp_sampled_path for ifg_path in params[cf.INTERFEROGRAM_FILES] ] # perform some checks on existing ifgs log.debug('Checking APS correction status') if mpiops.run_once(shared.check_correction_status, ifg_paths, ifc.PYRATE_APS_ERROR): log.debug('Finished APS correction') return # return if True condition returned aps_error_files_on_disc = [ MultiplePaths.aps_error_path(i, params) for i in ifg_paths ] if all(a.exists() for a in aps_error_files_on_disc): log.warning("Reusing APS errors from previous run!!!") for ifg_path, a in mpiops.array_split( list(zip(ifg_paths, aps_error_files_on_disc))): phase = np.load(a) _save_aps_corrected_phase(ifg_path, phase) else: tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles) mpiops.comm.barrier() spatio_temporal_filter(tsincr, ifg_paths, params, preread_ifgs) mpiops.comm.barrier() shared.save_numpy_phase(ifg_paths, params)
def main(params: dict) -> None: """ PyRate merge main function. Assembles product tiles in to single geotiff files """ if params[C.SIGNAL_POLARITY] == 1: log.info( f"Saving output products with the same sign convention as input data" ) elif params[C.SIGNAL_POLARITY] == -1: log.info( f"Saving output products with reversed sign convention compared to the input data" ) else: log.warning(f"Check the value of signal_polarity parameter") out_types = [] tsfile = join(params[C.TMPDIR], 'tscuml_0.npy') if exists(tsfile): _merge_timeseries(params, 'tscuml') _merge_linrate(params) out_types += ['linear_rate', 'linear_error', 'linear_rsquared'] # optional save of merged tsincr products if params["savetsincr"] == 1: _merge_timeseries(params, 'tsincr') else: log.warning( 'Not merging time series products; {} does not exist'.format( tsfile)) stfile = join(params[C.TMPDIR], 'stack_rate_0.npy') if exists(stfile): # setup paths mpiops.run_once(_merge_stack, params) out_types += ['stack_rate', 'stack_error'] else: log.warning( 'Not merging stack products; {} does not exist'.format(stfile)) if len(out_types) > 0: process_out_types = mpiops.array_split(out_types) for out_type in process_out_types: create_png_and_kml_from_tif(params[C.VELOCITY_DIR], output_type=out_type) else: log.warning('Exiting: no products to merge')
def dem_error_calc_wrapper(params: dict) -> None: """ MPI wrapper for DEM error correction :param params: Dictionary of PyRate configuration parameters. """ if not params[C.DEMERROR]: log.info("DEM error correction not required") return # geometry information needed to calculate Bperp for each pixel using first IFG in list ifg_paths = [ ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES] ] # check if DEM error correction is already available if mpiops.run_once(__check_and_apply_demerrors_found_on_disc, ifg_paths, params): log.warning("Reusing DEM error correction from previous run!!!") else: log.info("Calculating DEM error correction") # read and open the first IFG in list ifg0_path = ifg_paths[0] ifg0 = Ifg(ifg0_path) ifg0.open(readonly=True) # not currently implemented for ROIPAC data which breaks some tests # if statement can be deleted once ROIPAC is deprecated from PyRate if ifg0.meta_data[ifc.PYRATE_INSAR_PROCESSOR] == 'ROIPAC': log.warning( "Geometry calculations are not implemented for ROI_PAC") return if params[C.BASE_FILE_LIST] is None: log.warning( "No baseline files supplied: DEM error has not been computed") return # todo: subtract other corrections (e.g. orbital) from displacement phase before estimating the DEM error # the following code is a quick way to do the bperp calculation, but is not identical to the GAMMA output # where the near range of the first SLC is used for each pair. # calculate look angle for interferograms (using the Near Range of the first SLC) # look_angle = geometry.calc_local_geometry(ifg0, None, rg, lon, lat, params) # bperp = geometry.calc_local_baseline(ifg0, az, look_angle) # split full arrays in to tiles for parallel processing tiles_split(_process_dem_error_per_tile, params) preread_ifgs = params[C.PREREAD_IFGS] # write dem error and correction values to file mpiops.run_once(_write_dem_errors, ifg_paths, params, preread_ifgs) shared.save_numpy_phase(ifg_paths, params) log.debug('Finished DEM error correction step')
def main(params: dict) -> None: """ PyRate merge main function. Assembles product tiles in to single geotiff files """ out_types = [] stfile = join(params[cf.TMPDIR], 'stack_rate_0.npy') if exists(stfile): # setup paths mpiops.run_once(_merge_stack, params) out_types += ['stack_rate', 'stack_error'] else: log.warning( 'Not merging stack products; {} does not exist'.format(stfile)) tsfile = join(params[cf.TMPDIR], 'tscuml_0.npy') if exists(tsfile): _merge_timeseries(params, 'tscuml') _merge_linrate(params) out_types += ['linear_rate', 'linear_error', 'linear_rsquared'] # optional save of merged tsincr products if params["savetsincr"] == 1: _merge_timeseries(params, 'tsincr') else: log.warning( 'Not merging time series products; {} does not exist'.format( tsfile)) if len(out_types) > 0: process_out_types = mpiops.array_split(out_types) for out_type in process_out_types: create_png_and_kml_from_tif(params[cf.OUT_DIR], output_type=out_type) else: log.warning('Exiting: no products to merge')
def network_orbital_correction(ifg_paths, params, m_ifgs: Optional[List] = None): """ This algorithm implements a network inversion to determine orbital corrections for a set of interferograms forming a connected network. Warning: This will write orbital error corrected phase_data to the ifgs. :param list ifg_paths: List of Ifg class objects reduced to a minimum spanning tree network :param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC) :param bool offset: True to calculate the model using offsets :param dict params: dictionary of configuration parameters :param list m_ifgs: list of multilooked Ifg class objects (sequence must be multilooked versions of 'ifgs' arg) :param dict preread_ifgs: Dictionary containing information specifically for MPI jobs (optional) :return: None - interferogram phase data is updated and saved to disk """ # pylint: disable=too-many-locals, too-many-arguments offset = params[cf.ORBFIT_OFFSET] degree = params[cf.ORBITAL_FIT_DEGREE] preread_ifgs = params[cf.PREREAD_IFGS] # all orbit corrections available? if isinstance(ifg_paths[0], str): if __check_and_apply_orberrors_found_on_disc(ifg_paths, params): log.warning("Reusing orbfit errors from previous run!!!") return # all corrections are available in numpy files already saved - return ifgs = [shared.Ifg(i) for i in ifg_paths] else: # alternate test paths # TODO: improve ifgs = ifg_paths src_ifgs = ifgs if m_ifgs is None else m_ifgs src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst vphase = vstack([i.phase_data.reshape((i.num_cells, 1)) for i in src_ifgs]) vphase = squeeze(vphase) B = get_network_design_matrix(src_ifgs, degree, offset) # filter NaNs out before getting model B = B[~isnan(vphase)] orbparams = dot(pinv(B, 1e-6), vphase[~isnan(vphase)]) ncoef = _get_num_params(degree) if preread_ifgs: temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values() ids = first_second_ids(get_all_epochs(temp_ifgs)) else: ids = first_second_ids(get_all_epochs(ifgs)) coefs = [ orbparams[i:i + ncoef] for i in range(0, len(set(ids)) * ncoef, ncoef) ] # create full res DM to expand determined coefficients into full res # orbital correction (eg. expand coarser model to full size) if preread_ifgs: temp_ifg = Ifg(ifg_paths[0]) # ifgs here are paths temp_ifg.open() dm = get_design_matrix(temp_ifg, degree, offset=False) temp_ifg.close() else: ifg = ifgs[0] dm = get_design_matrix(ifg, degree, offset=False) for i in ifg_paths: # open if not Ifg instance if isinstance(i, str): # pragma: no cover # are paths i = Ifg(i) i.open(readonly=False) shared.nan_and_mm_convert(i, params) _remove_network_orb_error(coefs, dm, i, ids, offset, params)
def _ref_pixel_calc(ifg_paths: List[str], params: dict) -> Tuple[int, int]: """ Wrapper for reference pixel calculation """ lon = params[cf.REFX] lat = params[cf.REFY] ifg = Ifg(ifg_paths[0]) ifg.open(readonly=True) # assume all interferograms have same projection and will share the same transform transform = ifg.dataset.GetGeoTransform() if lon == -1 or lat == -1: log.info('Searching for best reference pixel location') half_patch_size, thresh, grid = refpixel.ref_pixel_setup( ifg_paths, params) process_grid = mpiops.array_split(grid) refpixel.save_ref_pixel_blocks(process_grid, half_patch_size, ifg_paths, params) mean_sds = refpixel._ref_pixel_mpi(process_grid, half_patch_size, ifg_paths, thresh, params) mean_sds = mpiops.comm.gather(mean_sds, root=0) if mpiops.rank == MASTER_PROCESS: mean_sds = np.hstack(mean_sds) refpixel_returned = mpiops.run_once(refpixel.find_min_mean, mean_sds, grid) if isinstance(refpixel_returned, ValueError): from pyrate.core.refpixel import RefPixelError raise RefPixelError( "Reference pixel calculation returned an all nan slice!\n" "Cannot continue downstream computation. Please change reference pixel algorithm used before " "continuing.") refy, refx = refpixel_returned # row first means first value is latitude log.info('Selected reference pixel coordinate (x, y): ({}, {})'.format( refx, refy)) lon, lat = refpixel.convert_pixel_value_to_geographic_coordinate( refx, refy, transform) log.info( 'Selected reference pixel coordinate (lon, lat): ({}, {})'.format( lon, lat)) else: log.info('Using reference pixel from config file (lon, lat): ({}, {})'. format(lon, lat)) log.warning( "Ensure user supplied reference pixel values are in lon/lat") refx, refy = refpixel.convert_geographic_coordinate_to_pixel_value( lon, lat, transform) log.info( 'Converted reference pixel coordinate (x, y): ({}, {})'.format( refx, refy)) refpixel.update_refpix_metadata(ifg_paths, refx, refy, transform, params) log.debug("refpx, refpy: " + str(refx) + " " + str(refy)) ifg.close() return int(refx), int(refy)
def main(): parser = argparse.ArgumentParser( prog='plot_ifgs', description="Python script to plot interferograms", add_help=True, formatter_class=RawTextHelpFormatter) parser.add_argument('-v', '--verbosity', type=str, default='INFO', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'], help="Increase output verbosity") parser.add_argument('-d', '--directory', action="store", type=str, default=None, help="Pass path to directory containing ifgs", required=True) parser.add_argument('-f', '--config_file', action="store", type=str, default=None, help="Pass configuration file", required=True) parser.add_argument('-n', '--ifgs_per_plot', type=int, default=50, help='number of ifgs per plot', required=False) args = parser.parse_args() params = _params_from_conf(args.config_file) configure_stage_log( args.verbosity, 'plot_ifgs', Path(params[C.OUT_DIR]).joinpath('pyrate.log.').as_posix()) if args.verbosity: log.setLevel(args.verbosity) log.info("Verbosity set to " + str(args.verbosity) + ".") log.debug("Arguments supplied at command line: ") log.debug(args) ifgs = sorted(list(Path(args.directory).glob('*_ifg.tif'))) num_ifgs = len(ifgs) if num_ifgs == 0: log.warning( f'No interferograms with extension *_ifg.tif were found in {args.directory}' ) return log.info(f'Plotting {num_ifgs} interferograms found in {args.directory}') ifgs_per_plot = args.ifgs_per_plot plt_rows = np.int(np.sqrt(ifgs_per_plot)) plt_cols = ifgs_per_plot // plt_rows if ifgs_per_plot % plt_rows: plt_cols += 1 tot_plots = 0 num_of_figs = math.ceil(num_ifgs / ifgs_per_plot) fig_no = 0 for i in range(num_of_figs): fig_no += 1 fig = plt.figure(figsize=(12 * plt_rows, 8 * plt_cols)) fig_plots = 0 for p_r in range(plt_rows): for p_c in range(plt_cols): ax = fig.add_subplot(plt_rows, plt_cols, fig_plots + 1) ifg_num = plt_cols * p_r + p_c + ifgs_per_plot * i file = ifgs[ifg_num] log.info(f'Plotting {file}') __plot_ifg(file, cmap, ax, num_ifgs, params) tot_plots += 1 fig_plots += 1 log.debug(f'Plotted interferogram #{tot_plots}') if (fig_plots == ifgs_per_plot) or (tot_plots == num_ifgs): f_name = Path(args.directory).joinpath('ifg-phase-plot-' + str(fig_no) + '.png').as_posix() plt.savefig(f_name, dpi=50) log.info(f'{fig_plots} interferograms plotted in {f_name}') plt.close(fig) break if tot_plots == num_ifgs: break
def ref_pixel_calc_wrapper(params: dict) -> Tuple[int, int]: """ Wrapper for reference pixel calculation """ __validate_supplied_lat_lon(params) ifg_paths = [ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]] lon = params[C.REFX] lat = params[C.REFY] ifg = Ifg(ifg_paths[0]) ifg.open(readonly=True) # assume all interferograms have same projection and will share the same transform transform = ifg.dataset.GetGeoTransform() ref_pixel_file = Configuration.ref_pixel_path(params) def __reuse_ref_pixel_file_if_exists(): if ref_pixel_file.exists(): refx, refy = np.load(ref_pixel_file) log.info('Reusing pre-calculated ref-pixel values: ({}, {}) from file {}'.format( refx, refy, ref_pixel_file.as_posix())) log.warning("Reusing ref-pixel values from previous run!!!") params[C.REFX_FOUND], params[C.REFY_FOUND] = int(refx), int(refy) return int(refx), int(refy) else: return None, None # read and return refx, refy = mpiops.run_once(__reuse_ref_pixel_file_if_exists) if (refx is not None) and (refy is not None): update_refpix_metadata(ifg_paths, int(refx), int(refy), transform, params) return refx, refy if lon == -1 or lat == -1: log.info('Searching for best reference pixel location') half_patch_size, thresh, grid = ref_pixel_setup(ifg_paths, params) process_grid = mpiops.array_split(grid) save_ref_pixel_blocks(process_grid, half_patch_size, ifg_paths, params) mean_sds = _ref_pixel_mpi(process_grid, half_patch_size, ifg_paths, thresh, params) mean_sds = mpiops.comm.gather(mean_sds, root=0) if mpiops.rank == MAIN_PROCESS: mean_sds = np.hstack(mean_sds) refpixel_returned = mpiops.run_once(find_min_mean, mean_sds, grid) if isinstance(refpixel_returned, ValueError): raise RefPixelError( "Reference pixel calculation returned an all nan slice!\n" "Cannot continue downstream computation. Please change reference pixel algorithm used before " "continuing.") refy, refx = refpixel_returned # row first means first value is latitude log.info('Selected reference pixel coordinate (x, y): ({}, {})'.format(refx, refy)) lon, lat = convert_pixel_value_to_geographic_coordinate(refx, refy, transform) log.info('Selected reference pixel coordinate (lon, lat): ({}, {})'.format(lon, lat)) else: log.info('Using reference pixel from config file (lon, lat): ({}, {})'.format(lon, lat)) log.warning("Ensure user supplied reference pixel values are in lon/lat") refx, refy = convert_geographic_coordinate_to_pixel_value(lon, lat, transform) log.info('Converted reference pixel coordinate (x, y): ({}, {})'.format(refx, refy)) np.save(file=ref_pixel_file, arr=[int(refx), int(refy)]) update_refpix_metadata(ifg_paths, refx, refy, transform, params) log.debug("refpx, refpy: "+str(refx) + " " + str(refy)) ifg.close() params[C.REFX_FOUND], params[C.REFY_FOUND] = int(refx), int(refy) return int(refx), int(refy)
def ref_phase_est_wrapper(params): """ Wrapper for reference phase estimation. """ ifg_paths = [ ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES] ] refpx, refpy = params[C.REFX_FOUND], params[C.REFY_FOUND] if len(ifg_paths) < 2: raise ReferencePhaseError( "At least two interferograms required for reference phase correction ({len_ifg_paths} " "provided).".format(len_ifg_paths=len(ifg_paths))) # this is not going to be true as we now start with fresh multilooked ifg copies - remove? if mpiops.run_once(shared.check_correction_status, ifg_paths, ifc.PYRATE_REF_PHASE): log.warning( 'Reference phase correction already applied to ifgs; returning') return ifgs = [Ifg(ifg_path) for ifg_path in ifg_paths] # Save reference phase numpy arrays to disk. ref_phs_file = Configuration.ref_phs_file(params) # If ref phase file exists on disk, then reuse - subtract ref_phase from ifgs and return if ref_phs_file.exists(): ref_phs = np.load(ref_phs_file) _update_phase_and_metadata(ifgs, ref_phs, params) shared.save_numpy_phase(ifg_paths, params) return ref_phs, ifgs # determine the reference phase for each ifg if params[C.REF_EST_METHOD] == 1: log.info("Calculating reference phase as median of interferogram") ref_phs = est_ref_phase_ifg_median(ifg_paths, params) elif params[C.REF_EST_METHOD] == 2: log.info( 'Calculating reference phase in a patch surrounding pixel (x, y): ({}, {})' .format(refpx, refpy)) ref_phs = est_ref_phase_patch_median(ifg_paths, params, refpx, refpy) else: raise ReferencePhaseError( "No such option, set parameter 'refest' to '1' or '2'.") # gather all reference phases from distributed processes and save to disk if mpiops.rank == MAIN_PROCESS: collected_ref_phs = np.zeros(len(ifg_paths), dtype=np.float64) process_indices = mpiops.array_split(range(len(ifg_paths))).astype( np.uint16) collected_ref_phs[process_indices] = ref_phs for r in range(1, mpiops.size): process_indices = mpiops.array_split(range(len(ifg_paths)), r).astype(np.uint16) this_process_ref_phs = np.zeros(shape=len(process_indices), dtype=np.float64) mpiops.comm.Recv(this_process_ref_phs, source=r, tag=r) collected_ref_phs[process_indices] = this_process_ref_phs np.save(file=ref_phs_file, arr=collected_ref_phs) else: collected_ref_phs = np.empty(len(ifg_paths), dtype=np.float64) mpiops.comm.Send(ref_phs, dest=MAIN_PROCESS, tag=mpiops.rank) mpiops.comm.Bcast(collected_ref_phs, root=0) # subtract ref_phase from ifgs _update_phase_and_metadata(ifgs, collected_ref_phs, params) mpiops.comm.barrier() shared.save_numpy_phase(ifg_paths, params) log.debug("Finished reference phase correction") # Preserve old return value so tests don't break. return ref_phs, ifgs
def network_orbital_correction(ifg_paths, params, m_ifgs: Optional[List] = None): """ This algorithm implements a network inversion to determine orbital corrections for a set of interferograms forming a connected network. Warning: This will write orbital error corrected phase_data to the ifgs. :param list ifg_paths: List of Ifg class objects reduced to a minimum spanning tree network :param dict params: dictionary of configuration parameters :param list m_ifgs: list of multilooked Ifg class objects (sequence must be multilooked versions of 'ifgs' arg) :return: None - interferogram phase data is updated and saved to disk """ # pylint: disable=too-many-locals, too-many-arguments offset = params[C.ORBFIT_OFFSET] degree = params[C.ORBITAL_FIT_DEGREE] preread_ifgs = params[C.PREREAD_IFGS] intercept = params[C.ORBFIT_INTERCEPT] scale = params[C.ORBFIT_SCALE] # all orbit corrections available? if isinstance(ifg_paths[0], str): if __check_and_apply_orberrors_found_on_disc(ifg_paths, params): log.warning("Reusing orbfit errors from previous run!!!") return # all corrections are available in numpy files already saved - return ifgs = [shared.Ifg(i) for i in ifg_paths] else: # alternate test paths # TODO: improve ifgs = ifg_paths src_ifgs = ifgs if m_ifgs is None else m_ifgs src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst if preread_ifgs: temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values() ids = first_second_ids(get_all_epochs(temp_ifgs)) else: ids = first_second_ids(get_all_epochs(ifgs)) nepochs = len(set(ids)) # call the actual inversion routine coefs = calc_network_orb_correction(src_ifgs, degree, scale, nepochs, intercept=intercept) # create full res DM to expand determined coefficients into full res # orbital correction (eg. expand coarser model to full size) if preread_ifgs: temp_ifg = Ifg(ifg_paths[0]) # ifgs here are paths temp_ifg.open() dm = get_design_matrix(temp_ifg, degree, intercept=intercept, scale=scale) temp_ifg.close() else: ifg = ifgs[0] dm = get_design_matrix(ifg, degree, intercept=intercept, scale=scale) for i in ifg_paths: # open if not Ifg instance if isinstance(i, str): # pragma: no cover # are paths i = Ifg(i) i.open(readonly=False) shared.nan_and_mm_convert(i, params) _remove_network_orb_error(coefs, dm, i, ids, offset, params)