def test_single_workflow(gamma_conf): check_call(f"mpirun -n 4 pyrate workflow -f {gamma_conf}", shell=True) params = Configuration(gamma_conf).__dict__ log_file_name = 'pyrate.log.' + 'workflow' files = list(Path(params[cf.OUT_DIR]).glob(log_file_name + '.*')) assert len(files) == 1 # ref pixel file generated ref_pixel_file = params[cf.REF_PIXEL_FILE] assert Path(ref_pixel_file).exists() ref_pixel = np.load(ref_pixel_file) np.testing.assert_array_equal(ref_pixel, [38, 58]) # assert orbfit exists on disc from pyrate.core import shared looked_files = [p.sampled_path for p in params[cf.INTERFEROGRAM_FILES]] ifgs = [shared.Ifg(ifg) for ifg in looked_files] orbfits_on_disc = [ Path(params[cf.OUT_DIR], cf.ORB_ERROR_DIR, Path(ifg.data_path).stem + '_orbfit.npy') for ifg in ifgs ] assert all(orbfits_on_disc) shutil.rmtree(params[cf.OUT_DIR])
def setup_class(cls): params = Configuration(common.TEST_CONF_ROIPAC).__dict__ cls.tmp_dir = tempfile.mkdtemp() common.copytree(common.SML_TEST_TIF, cls.tmp_dir) tifs = glob.glob(os.path.join(cls.tmp_dir, "*.tif")) for t in tifs: os.chmod(t, 0o644) small_ifgs = common.small_data_setup(datafiles=tifs) ifg_paths = [i.data_path for i in small_ifgs] cls.ifg_ret = common.pre_prepare_ifgs(ifg_paths, params=params) for i in cls.ifg_ret: i.close() nan_conversion = params[C.NAN_CONVERSION] # prepare a second set cls.tmp_dir2 = tempfile.mkdtemp() common.copytree(common.SML_TEST_TIF, cls.tmp_dir2) tifs = glob.glob(os.path.join(cls.tmp_dir2, "*.tif")) for t in tifs: os.chmod(t, 0o644) small_ifgs = common.small_data_setup(datafiles=tifs) ifg_paths = [i.data_path for i in small_ifgs] cls.ifgs = [shared.Ifg(p) for p in ifg_paths] for i in cls.ifgs: i.open(readonly=False) if nan_conversion: # nan conversion happens here in networkx mst i.nodata_value = params[C.NO_DATA_VALUE] i.convert_to_nans() if not i.mm_converted: i.convert_to_mm() i.close()
def test_single_workflow(gamma_or_mexicoa_conf): if MPI_INSTALLED: check_call(f"mpirun -n 4 pyrate workflow -f {gamma_or_mexicoa_conf}", shell=True) else: check_call(f"pyrate workflow -f {gamma_or_mexicoa_conf}", shell=True) params = Configuration(gamma_or_mexicoa_conf).__dict__ log_file_name = 'pyrate.log.' + 'workflow' files = list(Path(params[C.OUT_DIR]).glob(log_file_name + '.*')) assert len(files) == 1 # ref pixel file generated ref_pixel_file = params[C.REF_PIXEL_FILE] assert Path(ref_pixel_file).exists() ref_pixel = np.load(ref_pixel_file) if gamma_or_mexicoa_conf == MEXICO_CROPA_CONF: np.testing.assert_array_equal(ref_pixel, [42, 2]) for f in C.GEOMETRY_OUTPUT_TYPES: assert Path(params[C.GEOMETRY_DIR]).joinpath(f + '.tif').exists() else: np.testing.assert_array_equal(ref_pixel, [38, 58]) # assert orbfit exists on disc from pyrate.core import shared looked_files = [p.sampled_path for p in params[C.INTERFEROGRAM_FILES]] ifgs = [shared.Ifg(ifg) for ifg in looked_files] orbfits_on_disc = [ Path(params[C.OUT_DIR], C.ORB_ERROR_DIR, Path(ifg.data_path).stem + '_orbfit.npy') for ifg in ifgs ] assert all(orbfits_on_disc) shutil.rmtree(params[C.OUT_DIR])
def get_ifgs(out_dir, _open=True): paths = glob.glob(join(out_dir, 'geo_*-*_unw.tif')) ifgs = [shared.Ifg(p) for p in paths] assert len(ifgs) == 17, 'Got %s' % ifgs if _open: for i in ifgs: i.open(readonly=False) return ifgs
def cvd(ifg_path, params, r_dist, calc_alpha=False, write_vals=False, save_acg=False): """ Calculate the 1D covariance function of an entire interferogram as the radial average of its 2D autocorrelation. :param str ifg_path: An interferogram file path. OR :param dict params: Dictionary of configuration parameters :param ndarray r_dist: Array of distance values from the image centre (See Rdist class for more details) :param bool calc_alpha: If True calculate alpha :param bool write_vals: If True write maxvar and alpha values to interferogram metadata :param bool save_acg: If True write autocorrelation and radial distance data to numpy array file on disk :return: maxvar: The maximum variance (at zero lag) :rtype: float :return: alpha: the exponential length-scale of decay factor :rtype: float """ ifg = shared.Ifg(ifg_path) ifg.open() shared.nan_and_mm_convert(ifg, params) # calculate 2D auto-correlation of image using the # spectral method (Wiener-Khinchin theorem) if ifg.nan_converted: # if nancoverted earlier, convert nans back to 0's phase = where(isnan(ifg.phase_data), 0, ifg.phase_data) else: phase = ifg.phase_data maxvar, alpha = cvd_from_phase(phase, ifg, r_dist, calc_alpha, save_acg=save_acg, params=params) if write_vals: ifg.add_metadata(**{ ifc.PYRATE_MAXVAR: str(maxvar), ifc.PYRATE_ALPHA: str(alpha) }) ifg.close() return maxvar, alpha
def test_metadata(self): refx, refy = pyrate.core.refpixel.ref_pixel_calc_wrapper( self.params_chipsize_15) for i in self.ifg_paths: ifg = shared.Ifg(i) ifg.open(readonly=True) md = ifg.meta_data for k, v in zip([ ifc.PYRATE_REFPIX_X, ifc.PYRATE_REFPIX_Y, ifc.PYRATE_REFPIX_LAT, ifc.PYRATE_REFPIX_LON, ifc.PYRATE_MEAN_REF_AREA, ifc.PYRATE_STDDEV_REF_AREA ], [str(refx), str(refy), 0, 0, 0, 0]): assert k in md # metadata present # assert values ifg.close()
def setup_method(cls): cls.conf = common.TEST_CONF_GAMMA params = Configuration(cls.conf).__dict__ conv2tif.main(params) params = Configuration(cls.conf).__dict__ prepifg.main(params) cls.params = Configuration(cls.conf).__dict__ correct._copy_mlooked(cls.params) correct._update_params_with_tiles(cls.params) correct._create_ifg_dict(cls.params) multi_paths = cls.params[cf.INTERFEROGRAM_FILES] cls.ifg_paths = [p.tmp_sampled_path for p in multi_paths] cls.ifgs = [shared.Ifg(i) for i in cls.ifg_paths] for i in cls.ifgs: i.open() shared.save_numpy_phase(cls.ifg_paths, cls.params) correct.mst_calc_wrapper(cls.params)
def remove_orbital_error(ifgs: List, params: dict) -> None: """ Wrapper function for PyRate orbital error removal functionality. NB: the ifg data is modified in situ, rather than create intermediate files. The network method assumes the given ifgs have already been reduced to a minimum spanning tree network. """ mpiops.run_once(__orb_params_check, params) ifg_paths = [i.data_path for i in ifgs] if isinstance(ifgs[0], Ifg) else ifgs method = params[cf.ORBITAL_FIT_METHOD] # mlooking is not necessary for independent correction in a computational sense # can use multiple procesing if write_to_disc=True if method == INDEPENDENT_METHOD: log.info('Calculating orbital correction using independent method') #TODO: implement multi-looking for independent orbit method if params[cf.ORBITAL_FIT_LOOKS_X] > 1 or params[ cf.ORBITAL_FIT_LOOKS_Y] > 1: log.warning( 'Multi-looking is not applied in independent orbit method') ifgs = [shared.Ifg(p) for p in ifg_paths] if isinstance(ifgs[0], str) else ifgs process_ifgs = mpiops.array_split(ifgs) for ifg in process_ifgs: independent_orbital_correction(ifg, params=params) elif method == NETWORK_METHOD: log.info('Calculating orbital correction using network method') # Here we do all the multilooking in one process, but in memory # can use multiple processes if we write data to disc during # remove_orbital_error step # A performance comparison should be made for saving multilooked # files on disc vs in memory single process multilooking if mpiops.rank == MAIN_PROCESS: mlooked = __create_multilooked_dataset_for_network_correction( params) _validate_mlooked(mlooked, ifg_paths) network_orbital_correction(ifg_paths, params, mlooked) else: raise OrbitalError("Unrecognised orbital correction method")
def independent_orbital_correction(ifg, degree, offset, params): """ Calculates and removes an orbital error surface from a single independent interferogram. Warning: This will write orbital error corrected phase_data to the ifg. :param Ifg class instance ifg: the interferogram to be corrected :param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC) :param bool offset: True to calculate the model using an offset :param dict params: dictionary of configuration parameters :return: None - interferogram phase data is updated and saved to disk """ ifg = shared.Ifg(ifg) if isinstance(ifg, str) else ifg if not ifg.is_open: ifg.open() shared.nan_and_mm_convert(ifg, params) # vectorise, keeping NODATA vphase = reshape(ifg.phase_data, ifg.num_cells) dm = get_design_matrix(ifg, degree, offset) # filter NaNs out before getting model clean_dm = dm[~isnan(vphase)] data = vphase[~isnan(vphase)] model = lstsq(clean_dm, data)[0] # first arg is the model params # calculate forward model & morph back to 2D if offset: fullorb = np.reshape(np.dot(dm[:, :-1], model[:-1]), ifg.phase_data.shape) else: fullorb = np.reshape(np.dot(dm, model), ifg.phase_data.shape) offset_removal = nanmedian(np.ravel(ifg.phase_data - fullorb)) # subtract orbital error from the ifg ifg.phase_data -= (fullorb - offset_removal) # set orbfit meta tag and save phase to file _save_orbital_error_corrected_phase(ifg) if ifg.open(): ifg.close()
def network_orbital_correction(ifg_paths, params, m_ifgs: Optional[List] = None): """ This algorithm implements a network inversion to determine orbital corrections for a set of interferograms forming a connected network. Warning: This will write orbital error corrected phase_data to the ifgs. :param list ifg_paths: List of Ifg class objects reduced to a minimum spanning tree network :param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC) :param bool offset: True to calculate the model using offsets :param dict params: dictionary of configuration parameters :param list m_ifgs: list of multilooked Ifg class objects (sequence must be multilooked versions of 'ifgs' arg) :param dict preread_ifgs: Dictionary containing information specifically for MPI jobs (optional) :return: None - interferogram phase data is updated and saved to disk """ # pylint: disable=too-many-locals, too-many-arguments offset = params[cf.ORBFIT_OFFSET] degree = params[cf.ORBITAL_FIT_DEGREE] preread_ifgs = params[cf.PREREAD_IFGS] # all orbit corrections available? if isinstance(ifg_paths[0], str): if __check_and_apply_orberrors_found_on_disc(ifg_paths, params): log.warning("Reusing orbfit errors from previous run!!!") return # all corrections are available in numpy files already saved - return ifgs = [shared.Ifg(i) for i in ifg_paths] else: # alternate test paths # TODO: improve ifgs = ifg_paths src_ifgs = ifgs if m_ifgs is None else m_ifgs src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst vphase = vstack([i.phase_data.reshape((i.num_cells, 1)) for i in src_ifgs]) vphase = squeeze(vphase) B = get_network_design_matrix(src_ifgs, degree, offset) # filter NaNs out before getting model B = B[~isnan(vphase)] orbparams = dot(pinv(B, 1e-6), vphase[~isnan(vphase)]) ncoef = _get_num_params(degree) if preread_ifgs: temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values() ids = first_second_ids(get_all_epochs(temp_ifgs)) else: ids = first_second_ids(get_all_epochs(ifgs)) coefs = [ orbparams[i:i + ncoef] for i in range(0, len(set(ids)) * ncoef, ncoef) ] # create full res DM to expand determined coefficients into full res # orbital correction (eg. expand coarser model to full size) if preread_ifgs: temp_ifg = Ifg(ifg_paths[0]) # ifgs here are paths temp_ifg.open() dm = get_design_matrix(temp_ifg, degree, offset=False) temp_ifg.close() else: ifg = ifgs[0] dm = get_design_matrix(ifg, degree, offset=False) for i in ifg_paths: # open if not Ifg instance if isinstance(i, str): # pragma: no cover # are paths i = Ifg(i) i.open(readonly=False) shared.nan_and_mm_convert(i, params) _remove_network_orb_error(coefs, dm, i, ids, offset, params)
def network_orbital_correction(ifg_paths, params, m_ifgs: Optional[List] = None): """ This algorithm implements a network inversion to determine orbital corrections for a set of interferograms forming a connected network. Warning: This will write orbital error corrected phase_data to the ifgs. :param list ifg_paths: List of Ifg class objects reduced to a minimum spanning tree network :param dict params: dictionary of configuration parameters :param list m_ifgs: list of multilooked Ifg class objects (sequence must be multilooked versions of 'ifgs' arg) :return: None - interferogram phase data is updated and saved to disk """ # pylint: disable=too-many-locals, too-many-arguments offset = params[C.ORBFIT_OFFSET] degree = params[C.ORBITAL_FIT_DEGREE] preread_ifgs = params[C.PREREAD_IFGS] intercept = params[C.ORBFIT_INTERCEPT] scale = params[C.ORBFIT_SCALE] # all orbit corrections available? if isinstance(ifg_paths[0], str): if __check_and_apply_orberrors_found_on_disc(ifg_paths, params): log.warning("Reusing orbfit errors from previous run!!!") return # all corrections are available in numpy files already saved - return ifgs = [shared.Ifg(i) for i in ifg_paths] else: # alternate test paths # TODO: improve ifgs = ifg_paths src_ifgs = ifgs if m_ifgs is None else m_ifgs src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst if preread_ifgs: temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values() ids = first_second_ids(get_all_epochs(temp_ifgs)) else: ids = first_second_ids(get_all_epochs(ifgs)) nepochs = len(set(ids)) # call the actual inversion routine coefs = calc_network_orb_correction(src_ifgs, degree, scale, nepochs, intercept=intercept) # create full res DM to expand determined coefficients into full res # orbital correction (eg. expand coarser model to full size) if preread_ifgs: temp_ifg = Ifg(ifg_paths[0]) # ifgs here are paths temp_ifg.open() dm = get_design_matrix(temp_ifg, degree, intercept=intercept, scale=scale) temp_ifg.close() else: ifg = ifgs[0] dm = get_design_matrix(ifg, degree, intercept=intercept, scale=scale) for i in ifg_paths: # open if not Ifg instance if isinstance(i, str): # pragma: no cover # are paths i = Ifg(i) i.open(readonly=False) shared.nan_and_mm_convert(i, params) _remove_network_orb_error(coefs, dm, i, ids, offset, params)
def independent_orbital_correction(ifg_path, params): """ Calculates and removes an orbital error surface from a single independent interferogram. Warning: This will write orbital error corrected phase_data to the ifg. :param Ifg class instance ifg: the interferogram to be corrected :param dict params: dictionary of configuration parameters :return: None - interferogram phase data is updated and saved to disk """ log.debug(f"Orbital correction of {ifg_path}") degree = params[C.ORBITAL_FIT_DEGREE] offset = params[C.ORBFIT_OFFSET] intercept = params[C.ORBFIT_INTERCEPT] xlooks = params[C.ORBITAL_FIT_LOOKS_X] ylooks = params[C.ORBITAL_FIT_LOOKS_Y] ifg0 = shared.Ifg(ifg_path) if isinstance(ifg_path, str) else ifg_path # get full-resolution design matrix fullres_dm = get_design_matrix(ifg0, degree, intercept=intercept) ifg = shared.dem_or_ifg(ifg_path) if isinstance(ifg_path, str) else ifg_path multi_path = MultiplePaths(ifg.data_path, params) orb_on_disc = MultiplePaths.orb_error_path(ifg.data_path, params) if not ifg.is_open: ifg.open() shared.nan_and_mm_convert(ifg, params) fullres_ifg = ifg # keep a backup fullres_phase = fullres_ifg.phase_data if orb_on_disc.exists(): log.info( f'Reusing already computed orbital fit correction: {orb_on_disc}') orbital_correction = np.load(file=orb_on_disc) else: # Multi-look the ifg data if either X or Y is greater than 1 if (xlooks > 1) or (ylooks > 1): exts, _, _ = __extents_from_params(params) mlooked = _create_mlooked_dataset(multi_path, ifg.data_path, exts, params) ifg = Ifg(mlooked) # multi-looked Ifg object ifg.initialize() shared.nan_and_mm_convert(ifg, params) # vectorise phase data, keeping NODATA vphase = reshape(ifg.phase_data, ifg.num_cells) # compute design matrix for multi-looked data mlooked_dm = get_design_matrix(ifg, degree, intercept=intercept) # invert to obtain the correction image (forward model) at full-res orbital_correction = __orb_correction(fullres_dm, mlooked_dm, fullres_phase, vphase, offset=offset) # save correction to disc if not orb_on_disc.parent.exists(): shared.mkdir_p(orb_on_disc.parent) np.save(file=orb_on_disc, arr=orbital_correction) # subtract orbital correction from the full-res ifg fullres_ifg.phase_data -= orbital_correction # set orbfit meta tag and save phase to file _save_orbital_error_corrected_phase(fullres_ifg, params)