def test_plot_closure(mexico_cropa_params): config = Configuration(MEXICO_CROPA_CONF) params = config.__dict__ check_call(f"mpirun -n 3 pyrate prepifg -f {MEXICO_CROPA_CONF}", shell=True) correct._copy_mlooked(params) correct.__validate_correct_steps(params) # house keeping correct._update_params_with_tiles(params) correct._create_ifg_dict(params) params[C.REFX_FOUND], params[ C.REFY_FOUND] = correct.ref_pixel_calc_wrapper(params) # run through the correct steps in user specified sequence for step in steps: if step == 'phase_closure': correct.correct_steps[step](params, config) else: correct.correct_steps[step](params) closure_plot_file = Path(config.phase_closure_dir).joinpath( f'closure_loops_iteration_1_fig_0.png') assert closure_plot_file.exists() shutil.rmtree(params[C.OUT_DIR], ignore_errors=True)
def test_dem_error_used_from_disc_on_rerun(self): correct._copy_mlooked(self.params) correct._update_params_with_tiles(self.params) times_written = self.__run_once() assert len(times_written) == len(self.ifg_paths) times_written_1 = self.__run_once() np.testing.assert_array_equal(times_written_1, times_written)
def setup_method(self): self.params = Configuration(common.TEST_CONF_GAMMA).__dict__ self.tmp_dir = tempfile.mkdtemp() self.params[cf.OUT_DIR] = self.tmp_dir self.params[cf.REF_EST_METHOD] = 1 self.params[cf.PARALLEL] = False self.params[cf.TMPDIR] = self.tmp_dir common.copytree(common.SML_TEST_TIF, self.tmp_dir) self.small_tifs = glob.glob(os.path.join(self.tmp_dir, "*.tif")) for s in self.small_tifs: os.chmod(s, 0o644) self.ifgs = common.small_data_setup(self.tmp_dir, is_dir=True) self.params[cf.INTERFEROGRAM_FILES] = [ MultiplePaths(p, self.params) for p in self.small_tifs ] for p in self.params[cf.INTERFEROGRAM_FILES]: p.sampled_path = p.converted_path p.tmp_sampled_path = p.sampled_path for ifg in self.ifgs: ifg.close() self.params[cf.REFX], self.params[cf.REFY] = -1, -1 self.params[cf.REFNX], self.params[cf.REFNY] = 10, 10 self.params[cf.REF_CHIP_SIZE], self.params[cf.REF_MIN_FRAC] = 21, 0.5 self.params['rows'], self.params['cols'] = 3, 2 self.params[cf.REF_PIXEL_FILE] = Configuration.ref_pixel_path( self.params) correct._update_params_with_tiles(self.params) correct.ref_pixel_calc_wrapper(self.params)
def setup_class(cls): params = Configuration(common.TEST_CONF_ROIPAC).__dict__ params[cf.TEMP_MLOOKED_DIR] = os.path.join(params[cf.OUT_DIR], cf.TEMP_MLOOKED_DIR) conv2tif.main(params) prepifg.main(params) params[cf.REF_EST_METHOD] = 2 xlks, _, crop = cf.transform_params(params) dest_paths, headers = common.repair_params_for_correct_tests(params[cf.OUT_DIR], params) correct._copy_mlooked(params) copied_dest_paths = [os.path.join(params[cf.TEMP_MLOOKED_DIR], os.path.basename(d)) for d in dest_paths] del dest_paths # start run_pyrate copy ifgs = pre_prepare_ifgs(copied_dest_paths, params) mst_grid = tests.common.mst_calculation(copied_dest_paths, params) refx, refy = pyrate.core.refpixel.ref_pixel_calc_wrapper(params) params[cf.REFX] = refx params[cf.REFY] = refy params[cf.ORBFIT_OFFSET] = True # Estimate and remove orbit errors pyrate.core.orbital.remove_orbital_error(ifgs, params) ifgs = prepare_ifgs_without_phase(copied_dest_paths, params) for ifg in ifgs: ifg.close() correct._update_params_with_tiles(params) _, ifgs = pyrate.core.ref_phs_est.ref_phase_est_wrapper(params) ifgs[0].open() r_dist = vcm_module.RDist(ifgs[0])() ifgs[0].close() maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in copied_dest_paths] for ifg in ifgs: ifg.open() vcmt = vcm_module.get_vcmt(ifgs, maxvar) for ifg in ifgs: ifg.close() ifg.open() # Calculate stacked rate map params[cf.PARALLEL] = 1 cls.rate, cls.error, cls.samples = tests.common.calculate_stack_rate(ifgs, params, vcmt, mst_mat=mst_grid) # Calculate stacked rate map params[cf.PARALLEL] = 0 cls.rate_s, cls.error_s, cls.samples_s = tests.common.calculate_stack_rate(ifgs, params, vcmt, mst_mat=mst_grid) stackrate_dir = os.path.join(SML_TEST_DIR, 'stackrate') cls.rate_container = np.genfromtxt(os.path.join(stackrate_dir, 'stackmap.csv'), delimiter=',') cls.error_container = np.genfromtxt(os.path.join(stackrate_dir, 'errormap.csv'), delimiter=',') cls.samples_container = np.genfromtxt(os.path.join(stackrate_dir, 'coh_sta.csv'), delimiter=',') for ifg in ifgs: ifg.close() cls.params = params
def test_vcm_legacy_vs_mpi(mpisync, tempdir, roipac_or_gamma_conf): params = configuration.Configuration(roipac_or_gamma_conf).__dict__ LEGACY_VCM_DIR = os.path.join(SML_TEST_DIR, 'vcm') legacy_vcm = np.genfromtxt(os.path.join(LEGACY_VCM_DIR, 'vcmt.csv'), delimiter=',') tmpdir = Path(mpiops.run_once(tempdir)) params[C.OUT_DIR] = tmpdir.joinpath('out') params[C.PARALLEL] = 0 output_conf = Path(tmpdir).joinpath('conf.cfg') pyrate.configuration.write_config_file(params=params, output_conf_file=output_conf) params = configuration.Configuration(output_conf).__dict__ # dest_paths = [p.sampled_path for p in params[cf.INTERFEROGRAM_FILES]] # run conv2tif and prepifg, create the dest_paths files conv2tif.main(params) params = configuration.Configuration(output_conf).__dict__ prepifg.main(params) params = configuration.Configuration(output_conf).__dict__ params[C.ORBFIT_OFFSET] = True correct._copy_mlooked(params=params) correct._update_params_with_tiles(params) correct._create_ifg_dict(params=params) pyrate.core.refpixel.ref_pixel_calc_wrapper(params) pyrate.core.orbital.orb_fit_calc_wrapper(params) pyrate.core.ref_phs_est.ref_phase_est_wrapper(params) maxvar, vcmt = pyrate.core.covariance.maxvar_vcm_calc_wrapper(params) # phase data after ref pixel has changed due to commit bf2f7ebd # Legacy tests won't match anymore np.testing.assert_array_almost_equal(maxvar, legacy_maxvar, decimal=4) np.testing.assert_array_almost_equal(legacy_vcm, vcmt, decimal=3) mpiops.run_once(shutil.rmtree, tmpdir)
def update_params_due_to_ifg_selection(config): params = config.__dict__ if config.phase_closure_filtered_ifgs_list(params).exists(): params = config.refresh_ifg_list(params) correct._create_ifg_dict(params) correct._update_params_with_tiles(params) return params
def setup_class(cls): roipac_params = Configuration(TEST_CONF_ROIPAC).__dict__ from copy import deepcopy params = deepcopy(roipac_params) shared.mkdir_p(params[C.TMPDIR]) params[C.REF_EST_METHOD] = 2 conv2tif.main(params) params = deepcopy(roipac_params) prepifg.main(params) params = deepcopy(roipac_params) base_ifg_paths = [ c.unwrapped_path for c in params[C.INTERFEROGRAM_FILES] ] dest_paths = [c.converted_path for c in params[C.INTERFEROGRAM_FILES]] params[C.INTERFEROGRAM_FILES] = [ MultiplePaths(d, params) for d in dest_paths ] for p in params[C.INTERFEROGRAM_FILES]: # hack p.sampled_path = p.converted_path for i in dest_paths: Path(i).chmod( 0o664 ) # assign write permission as conv2tif output is readonly ifgs = common.pre_prepare_ifgs(dest_paths, params) correct._copy_mlooked(params) correct._update_params_with_tiles(params) correct._create_ifg_dict(params) pyrate.core.refpixel.ref_pixel_calc_wrapper(params) params[C.ORBFIT_OFFSET] = True pyrate.core.orbital.remove_orbital_error(ifgs, params) ifgs = prepare_ifgs_without_phase(dest_paths, params) for ifg in ifgs: ifg.close() for p in params[C.INTERFEROGRAM_FILES]: # hack p.tmp_sampled_path = p.sampled_path _, cls.ifgs = pyrate.core.ref_phs_est.ref_phase_est_wrapper(params) ifgs[0].open() r_dist = RDist(ifgs[0])() ifgs[0].close() # Calculate interferogram noise cls.maxvar = [ cvd(i, params, r_dist, calc_alpha=True, save_acg=True, write_vals=True)[0] for i in dest_paths ] cls.vcmt = get_vcmt(ifgs, cls.maxvar) for ifg in ifgs: ifg.close() cls.params = params
def __run_once(self): dem_files = [ MultiplePaths.dem_error_path(i, self.params) for i in self.ifg_paths ] correct._copy_mlooked(self.params) correct._update_params_with_tiles(self.params) correct._create_ifg_dict(self.params) save_numpy_phase(self.ifg_paths, self.params) dem_error_calc_wrapper(self.params) assert all(m.exists() for m in dem_files) return [os.stat(o).st_mtime for o in dem_files]
def test_calc_dem_errors(self): # validate output of current version of the code with saved files from an independent test run # only the reference phase and dem_error are used in this test # saved dem_error.tif (expected) dem_error_tif_exp = join(dem_error_path, 'dem_error.tif') dem = DEM(dem_error_tif_exp) dem_error_exp = dem.data # run relevant parts of the 'correct' step correct._copy_mlooked(self.params) correct._update_params_with_tiles(self.params) correct._create_ifg_dict(self.params) save_numpy_phase(self.ifg_paths, self.params) # subtract the reference phase to enable comparison with a 'normal' pyrate run ref_phase_est_wrapper(self.params) dem_error_calc_wrapper(self.params) # dem_error.tif from this run (result) dem_error_tif_res = join(self.params[C.DEM_ERROR_DIR], 'dem_error.tif') dem = DEM(dem_error_tif_res) dem_error_res = dem.data # check equality np.testing.assert_allclose(dem_error_exp, dem_error_res) # ifg correction files in subdirectory out/dem_error/ # three different ifgs: # ifg1 -> short_baseline_ifg: 20180106-20180319 (ca. 3 m) # ifg2 -> long_baseline_ifg: 20180130-20180412(ca. 108 m) # ifg3 -> medium_baseline_ifg: 20180412-20180518 (ca. 48 m) # load saved files dem_error_ifg1_path = join(dem_error_path, '20180106-20180319_ifg_20_dem_error.npy') dem_error_ifg1_exp = np.load(dem_error_ifg1_path) dem_error_ifg2_path = join(dem_error_path, '20180130-20180412_ifg_20_dem_error.npy') dem_error_ifg2_exp = np.load(dem_error_ifg2_path) dem_error_ifg3_path = join(dem_error_path, '20180412-20180518_ifg_20_dem_error.npy') dem_error_ifg3_exp = np.load(dem_error_ifg3_path) # load correction values saved from this run (result) dem_error_ifg1_path = Path(self.params[C.DEM_ERROR_DIR]).joinpath( '20180106-20180319_ifg_20_dem_error.npy') dem_error_ifg1_res = np.load(dem_error_ifg1_path) dem_error_ifg2_path = Path(self.params[C.DEM_ERROR_DIR]).joinpath( '20180130-20180412_ifg_20_dem_error.npy') dem_error_ifg2_res = np.load(dem_error_ifg2_path) dem_error_ifg3_path = Path(self.params[C.DEM_ERROR_DIR]).joinpath( '20180412-20180518_ifg_20_dem_error.npy') dem_error_ifg3_res = np.load(dem_error_ifg3_path) # check equality np.testing.assert_allclose(dem_error_ifg1_exp, dem_error_ifg1_res) np.testing.assert_allclose(dem_error_ifg2_exp, dem_error_ifg2_res) np.testing.assert_allclose(dem_error_ifg3_exp, dem_error_ifg3_res)
def setup_method(cls): cls.conf = common.TEST_CONF_GAMMA params = Configuration(cls.conf).__dict__ conv2tif.main(params) params = Configuration(cls.conf).__dict__ prepifg.main(params) cls.params = Configuration(cls.conf).__dict__ correct._copy_mlooked(cls.params) correct._update_params_with_tiles(cls.params) correct._create_ifg_dict(cls.params) multi_paths = cls.params[cf.INTERFEROGRAM_FILES] cls.ifg_paths = [p.tmp_sampled_path for p in multi_paths] cls.ifgs = [shared.Ifg(i) for i in cls.ifg_paths] for i in cls.ifgs: i.open() shared.save_numpy_phase(cls.ifg_paths, cls.params) correct.mst_calc_wrapper(cls.params)
def test_orbital_correction_legacy_equality(self): from pyrate import correct from pyrate.configuration import MultiplePaths multi_paths = [MultiplePaths(p, params=self.params) for p in self.ifg_paths] for m in multi_paths: # cheat m.sampled_path = m.converted_path self.params[cf.INTERFEROGRAM_FILES] = multi_paths self.params['rows'], self.params['cols'] = 2, 3 Path(self.BASE_DIR).joinpath('tmpdir').mkdir(exist_ok=True, parents=True) correct._copy_mlooked(self.params) correct._update_params_with_tiles(self.params) correct._create_ifg_dict(self.params) correct._copy_mlooked(self.params) pyrate.core.orbital.orb_fit_calc_wrapper(self.params) onlyfiles = [f for f in os.listdir(SML_TEST_LEGACY_ORBITAL_DIR) if os.path.isfile(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f)) and f.endswith('.csv') and f.__contains__('_method1_')] count = 0 for i, f in enumerate(onlyfiles): ifg_data = np.genfromtxt(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f), delimiter=',') for k, j in enumerate([m.tmp_sampled_path for m in multi_paths]): ifg = Ifg(j) ifg.open() print(f) print(j) if os.path.basename(j).split('_ifg.')[0] == os.path.basename(f).split( '_orb_planar_1lks_method1_geo_')[1].split('.')[0]: count += 1 # all numbers equal np.testing.assert_array_almost_equal(ifg_data, ifg.phase_data, decimal=2) # means must also be equal assert np.nanmean(ifg_data) == pytest.approx(np.nanmean(ifg.phase_data), abs=1e-2) # number of nans must equal assert np.sum(np.isnan(ifg_data)) == np.sum(np.isnan(ifg.phase_data)) ifg.close() # ensure that we have expected number of matches assert count == len(self.ifg_paths)
def test_ref_phase_used_from_disc_on_rerun(self, ref_est_method): self.params = Configuration(self.conf).__dict__ self.params[cf.REF_EST_METHOD] = ref_est_method correct._update_params_with_tiles(self.params) phase_prev, time_written = self.__run_once() # run again phase_now, time_written_1 = self.__run_once() # and once more phase_again, time_written_2 = self.__run_once() # assert no new file was written assert time_written_1 == time_written assert time_written_2 == time_written # assert phase data is unchanged after applying ref_ph correction from disc np.testing.assert_array_equal(phase_now, phase_prev) np.testing.assert_array_equal(phase_now, phase_again)
def setup_class(cls): params = Configuration(common.TEST_CONF_ROIPAC).__dict__ conv2tif.main(params) prepifg.main(params) for p in params[cf.INTERFEROGRAM_FILES]: # hack p.tmp_sampled_path = p.sampled_path Path(p.sampled_path).chmod( 0o664 ) # assign write permission as conv2tif output is readonly params[cf.REF_EST_METHOD] = 2 params[cf.PARALLEL] = False params[cf.ORBFIT_OFFSET] = True dest_paths, headers = common.repair_params_for_correct_tests( params[cf.OUT_DIR], params) # start run_pyrate copy ifgs = common.pre_prepare_ifgs(dest_paths, params) mst_grid = common.mst_calculation(dest_paths, params) # Estimate reference pixel location refx, refy = ref_pixel_calc_wrapper(params) # Estimate and remove orbit errors remove_orbital_error(ifgs, params) for i in ifgs: i.close() ifgs = common.pre_prepare_ifgs(dest_paths, params) for i in ifgs: i.close() for p in params[cf.INTERFEROGRAM_FILES]: p.tmp_sampled_path = p.sampled_path params[cf.REFX], params[cf.REFY] = refx, refy params['rows'], params['cols'] = 3, 2 correct._update_params_with_tiles(params) cls.ref_phs, cls.ifgs = ref_phase_est_wrapper(params) cls.params = params
def setup_class(cls): params = Configuration(common.TEST_CONF_ROIPAC).__dict__ params[cf.TEMP_MLOOKED_DIR] = os.path.join(params[cf.OUT_DIR], cf.TEMP_MLOOKED_DIR) conv2tif.main(params) prepifg.main(params) params[cf.REF_EST_METHOD] = 2 xlks, _, crop = cf.transform_params(params) dest_paths, headers = common.repair_params_for_correct_tests( params[cf.OUT_DIR], params) correct._copy_mlooked(params) copied_dest_paths = [ os.path.join(params[cf.TEMP_MLOOKED_DIR], os.path.basename(d)) for d in dest_paths ] del dest_paths # start run_pyrate copy ifgs = common.pre_prepare_ifgs(copied_dest_paths, params) mst_grid = common.mst_calculation(copied_dest_paths, params) refx, refy = pyrate.core.refpixel.ref_pixel_calc_wrapper(params) params[cf.REFX] = refx params[cf.REFY] = refy params[cf.ORBFIT_OFFSET] = True # Estimate and remove orbit errors pyrate.core.orbital.remove_orbital_error(ifgs, params) ifgs = common.prepare_ifgs_without_phase(copied_dest_paths, params) for ifg in ifgs: ifg.close() correct._update_params_with_tiles(params) _, ifgs = pyrate.core.ref_phs_est.ref_phase_est_wrapper(params) ifgs[0].open() r_dist = covariance.RDist(ifgs[0])() ifgs[0].close() # Calculate interferogram noise maxvar = [ covariance.cvd(i, params, r_dist)[0] for i in copied_dest_paths ] for ifg in ifgs: ifg.open() vcmt = covariance.get_vcmt(ifgs, maxvar) for ifg in ifgs: ifg.close() ifg.open() ifg.nodata_value = 0.0 params[cf.TIME_SERIES_METHOD] = 2 params[cf.PARALLEL] = 1 # Calculate time series cls.tsincr, cls.tscum, _ = common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid) params[cf.PARALLEL] = 0 # Calculate time series serailly by the pixel cls.tsincr_0, cls.tscum_0, _ = common.calculate_time_series( ifgs, params, vcmt, mst=mst_grid) # copy legacy data SML_TIME_SERIES_DIR = os.path.join(common.SML_TEST_DIR, 'time_series') tsincr_path = os.path.join(SML_TIME_SERIES_DIR, 'ts_incr_interp0_method2.csv') ts_incr = np.genfromtxt(tsincr_path) tscum_path = os.path.join(SML_TIME_SERIES_DIR, 'ts_cum_interp0_method2.csv') ts_cum = np.genfromtxt(tscum_path) cls.ts_incr = np.reshape(ts_incr, newshape=cls.tsincr_0.shape, order='F') cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F') cls.params = params
def test_ref_phase_used_from_disc_on_rerun(self): correct._update_params_with_tiles(self.params) times_written = self.__run_once() times_written_1 = self.__run_once() np.testing.assert_array_equal(times_written_1, times_written)