def test_plot_closure(mexico_cropa_params): config = Configuration(MEXICO_CROPA_CONF) params = config.__dict__ check_call(f"mpirun -n 3 pyrate prepifg -f {MEXICO_CROPA_CONF}", shell=True) correct._copy_mlooked(params) correct.__validate_correct_steps(params) # house keeping correct._update_params_with_tiles(params) correct._create_ifg_dict(params) params[C.REFX_FOUND], params[ C.REFY_FOUND] = correct.ref_pixel_calc_wrapper(params) # run through the correct steps in user specified sequence for step in steps: if step == 'phase_closure': correct.correct_steps[step](params, config) else: correct.correct_steps[step](params) closure_plot_file = Path(config.phase_closure_dir).joinpath( f'closure_loops_iteration_1_fig_0.png') assert closure_plot_file.exists() shutil.rmtree(params[C.OUT_DIR], ignore_errors=True)
def test_vcm_legacy_vs_mpi(mpisync, tempdir, roipac_or_gamma_conf): params = configuration.Configuration(roipac_or_gamma_conf).__dict__ LEGACY_VCM_DIR = os.path.join(SML_TEST_DIR, 'vcm') legacy_vcm = np.genfromtxt(os.path.join(LEGACY_VCM_DIR, 'vcmt.csv'), delimiter=',') tmpdir = Path(mpiops.run_once(tempdir)) params[C.OUT_DIR] = tmpdir.joinpath('out') params[C.PARALLEL] = 0 output_conf = Path(tmpdir).joinpath('conf.cfg') pyrate.configuration.write_config_file(params=params, output_conf_file=output_conf) params = configuration.Configuration(output_conf).__dict__ # dest_paths = [p.sampled_path for p in params[cf.INTERFEROGRAM_FILES]] # run conv2tif and prepifg, create the dest_paths files conv2tif.main(params) params = configuration.Configuration(output_conf).__dict__ prepifg.main(params) params = configuration.Configuration(output_conf).__dict__ params[C.ORBFIT_OFFSET] = True correct._copy_mlooked(params=params) correct._update_params_with_tiles(params) correct._create_ifg_dict(params=params) pyrate.core.refpixel.ref_pixel_calc_wrapper(params) pyrate.core.orbital.orb_fit_calc_wrapper(params) pyrate.core.ref_phs_est.ref_phase_est_wrapper(params) maxvar, vcmt = pyrate.core.covariance.maxvar_vcm_calc_wrapper(params) # phase data after ref pixel has changed due to commit bf2f7ebd # Legacy tests won't match anymore np.testing.assert_array_almost_equal(maxvar, legacy_maxvar, decimal=4) np.testing.assert_array_almost_equal(legacy_vcm, vcmt, decimal=3) mpiops.run_once(shutil.rmtree, tmpdir)
def test_orbital_error_method2_dummy(self): """ does not test anything except that the method is working """ # change to orbital error correction method 2 self.params[cf.ORBITAL_FIT_METHOD] = NETWORK_METHOD self.params[cf.ORBITAL_FIT_LOOKS_X] = 2 self.params[cf.ORBITAL_FIT_LOOKS_Y] = 2 correct._copy_mlooked(self.params) correct._create_ifg_dict(self.params) remove_orbital_error(self.new_data_paths, self.params) onlyfiles = [f for f in os.listdir(SML_TEST_LEGACY_ORBITAL_DIR) if os.path.isfile(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f)) and f.endswith('.csv') and f.__contains__('_method2_')] count = 0 for i, f in enumerate(onlyfiles): legacy_phase_data = np.genfromtxt(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f), delimiter=',') for k, j in enumerate(self.new_data_paths): if os.path.basename(j).split('_unw.')[0] == os.path.basename(f).split('_method2_')[1].split('.')[0]: count += 1 ifg = Ifg(j) ifg.open() # number of nans must equal assert np.sum(np.isnan(legacy_phase_data)) == np.sum(np.isnan(ifg.phase_data)) # ensure that we have expected number of matches assert count == len(self.new_data_paths)
def test_orbital_correction_legacy_equality_orbfit_method_2(self): correct._copy_mlooked(self.params) correct._create_ifg_dict(self.params) remove_orbital_error(self.new_data_paths, self.params) onlyfiles = [f for f in os.listdir(SML_TEST_LEGACY_ORBITAL_DIR) if os.path.isfile(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f)) and f.endswith('.csv') and f.__contains__('_method2_')] count = 0 for i, f in enumerate(onlyfiles): legacy_phase_data = np.genfromtxt(os.path.join( SML_TEST_LEGACY_ORBITAL_DIR, f), delimiter=',') for k, j in enumerate(self.new_data_paths): if os.path.basename(j).split('_unw.')[0] == os.path.basename(f).split('_method2_')[1].split('.')[0]: count += 1 ifg = Ifg(j) ifg.open() # all numbers equal # Note this changed as the nodata mask in the gdal_python.gdal_average changed to nan from 0 # np.testing.assert_array_almost_equal(legacy_phase_data, ifg.phase_data, decimal=3) # number of nans must equal assert np.sum(np.isnan(legacy_phase_data)) == np.sum(np.isnan(ifg.phase_data)) # ensure that we have expected number of matches assert count == len(self.new_data_paths)
def update_params_due_to_ifg_selection(config): params = config.__dict__ if config.phase_closure_filtered_ifgs_list(params).exists(): params = config.refresh_ifg_list(params) correct._create_ifg_dict(params) correct._update_params_with_tiles(params) return params
def setup_class(cls): cls.conf = TEST_CONF_GAMMA params = Configuration(cls.conf).__dict__ conv2tif.main(params) params = Configuration(cls.conf).__dict__ prepifg.main(params) cls.params = Configuration(cls.conf).__dict__ correct._copy_mlooked(cls.params) correct._create_ifg_dict(cls.params)
def __run_once(self): tiles = self.params[cf.TILES] mst_files = [Configuration.mst_path(self.params, t.index) for t in tiles] correct._copy_mlooked(self.params) correct._create_ifg_dict(self.params) save_numpy_phase(self.ifg_paths, self.params) mst.mst_calc_wrapper(self.params) assert all(m.exists() for m in mst_files) return [os.stat(o).st_mtime for o in mst_files]
def setup_method(cls): cls.conf = TEST_CONF_GAMMA params = Configuration(cls.conf).__dict__ conv2tif.main(params) params = Configuration(cls.conf).__dict__ prepifg.main(params) cls.params = Configuration(cls.conf).__dict__ correct._copy_mlooked(cls.params) correct._create_ifg_dict(cls.params) multi_paths = cls.params[cf.INTERFEROGRAM_FILES] cls.ifg_paths = [p.tmp_sampled_path for p in multi_paths]
def setup_class(cls): roipac_params = Configuration(TEST_CONF_ROIPAC).__dict__ from copy import deepcopy params = deepcopy(roipac_params) shared.mkdir_p(params[C.TMPDIR]) params[C.REF_EST_METHOD] = 2 conv2tif.main(params) params = deepcopy(roipac_params) prepifg.main(params) params = deepcopy(roipac_params) base_ifg_paths = [ c.unwrapped_path for c in params[C.INTERFEROGRAM_FILES] ] dest_paths = [c.converted_path for c in params[C.INTERFEROGRAM_FILES]] params[C.INTERFEROGRAM_FILES] = [ MultiplePaths(d, params) for d in dest_paths ] for p in params[C.INTERFEROGRAM_FILES]: # hack p.sampled_path = p.converted_path for i in dest_paths: Path(i).chmod( 0o664 ) # assign write permission as conv2tif output is readonly ifgs = common.pre_prepare_ifgs(dest_paths, params) correct._copy_mlooked(params) correct._update_params_with_tiles(params) correct._create_ifg_dict(params) pyrate.core.refpixel.ref_pixel_calc_wrapper(params) params[C.ORBFIT_OFFSET] = True pyrate.core.orbital.remove_orbital_error(ifgs, params) ifgs = prepare_ifgs_without_phase(dest_paths, params) for ifg in ifgs: ifg.close() for p in params[C.INTERFEROGRAM_FILES]: # hack p.tmp_sampled_path = p.sampled_path _, cls.ifgs = pyrate.core.ref_phs_est.ref_phase_est_wrapper(params) ifgs[0].open() r_dist = RDist(ifgs[0])() ifgs[0].close() # Calculate interferogram noise cls.maxvar = [ cvd(i, params, r_dist, calc_alpha=True, save_acg=True, write_vals=True)[0] for i in dest_paths ] cls.vcmt = get_vcmt(ifgs, cls.maxvar) for ifg in ifgs: ifg.close() cls.params = params
def __run_once(self): dem_files = [ MultiplePaths.dem_error_path(i, self.params) for i in self.ifg_paths ] correct._copy_mlooked(self.params) correct._update_params_with_tiles(self.params) correct._create_ifg_dict(self.params) save_numpy_phase(self.ifg_paths, self.params) dem_error_calc_wrapper(self.params) assert all(m.exists() for m in dem_files) return [os.stat(o).st_mtime for o in dem_files]
def test_calc_dem_errors(self): # validate output of current version of the code with saved files from an independent test run # only the reference phase and dem_error are used in this test # saved dem_error.tif (expected) dem_error_tif_exp = join(dem_error_path, 'dem_error.tif') dem = DEM(dem_error_tif_exp) dem_error_exp = dem.data # run relevant parts of the 'correct' step correct._copy_mlooked(self.params) correct._update_params_with_tiles(self.params) correct._create_ifg_dict(self.params) save_numpy_phase(self.ifg_paths, self.params) # subtract the reference phase to enable comparison with a 'normal' pyrate run ref_phase_est_wrapper(self.params) dem_error_calc_wrapper(self.params) # dem_error.tif from this run (result) dem_error_tif_res = join(self.params[C.DEM_ERROR_DIR], 'dem_error.tif') dem = DEM(dem_error_tif_res) dem_error_res = dem.data # check equality np.testing.assert_allclose(dem_error_exp, dem_error_res) # ifg correction files in subdirectory out/dem_error/ # three different ifgs: # ifg1 -> short_baseline_ifg: 20180106-20180319 (ca. 3 m) # ifg2 -> long_baseline_ifg: 20180130-20180412(ca. 108 m) # ifg3 -> medium_baseline_ifg: 20180412-20180518 (ca. 48 m) # load saved files dem_error_ifg1_path = join(dem_error_path, '20180106-20180319_ifg_20_dem_error.npy') dem_error_ifg1_exp = np.load(dem_error_ifg1_path) dem_error_ifg2_path = join(dem_error_path, '20180130-20180412_ifg_20_dem_error.npy') dem_error_ifg2_exp = np.load(dem_error_ifg2_path) dem_error_ifg3_path = join(dem_error_path, '20180412-20180518_ifg_20_dem_error.npy') dem_error_ifg3_exp = np.load(dem_error_ifg3_path) # load correction values saved from this run (result) dem_error_ifg1_path = Path(self.params[C.DEM_ERROR_DIR]).joinpath( '20180106-20180319_ifg_20_dem_error.npy') dem_error_ifg1_res = np.load(dem_error_ifg1_path) dem_error_ifg2_path = Path(self.params[C.DEM_ERROR_DIR]).joinpath( '20180130-20180412_ifg_20_dem_error.npy') dem_error_ifg2_res = np.load(dem_error_ifg2_path) dem_error_ifg3_path = Path(self.params[C.DEM_ERROR_DIR]).joinpath( '20180412-20180518_ifg_20_dem_error.npy') dem_error_ifg3_res = np.load(dem_error_ifg3_path) # check equality np.testing.assert_allclose(dem_error_ifg1_exp, dem_error_ifg1_res) np.testing.assert_allclose(dem_error_ifg2_exp, dem_error_ifg2_res) np.testing.assert_allclose(dem_error_ifg3_exp, dem_error_ifg3_res)
def setup_method(cls): cls.conf = common.TEST_CONF_GAMMA params = Configuration(cls.conf).__dict__ conv2tif.main(params) params = Configuration(cls.conf).__dict__ prepifg.main(params) cls.params = Configuration(cls.conf).__dict__ correct._copy_mlooked(cls.params) correct._update_params_with_tiles(cls.params) correct._create_ifg_dict(cls.params) multi_paths = cls.params[cf.INTERFEROGRAM_FILES] cls.ifg_paths = [p.tmp_sampled_path for p in multi_paths] cls.ifgs = [shared.Ifg(i) for i in cls.ifg_paths] for i in cls.ifgs: i.open() shared.save_numpy_phase(cls.ifg_paths, cls.params) correct.mst_calc_wrapper(cls.params)
def test_orbital_correction_legacy_equality(self): from pyrate import correct from pyrate.configuration import MultiplePaths multi_paths = [MultiplePaths(p, params=self.params) for p in self.ifg_paths] for m in multi_paths: # cheat m.sampled_path = m.converted_path self.params[cf.INTERFEROGRAM_FILES] = multi_paths self.params['rows'], self.params['cols'] = 2, 3 Path(self.BASE_DIR).joinpath('tmpdir').mkdir(exist_ok=True, parents=True) correct._copy_mlooked(self.params) correct._update_params_with_tiles(self.params) correct._create_ifg_dict(self.params) correct._copy_mlooked(self.params) pyrate.core.orbital.orb_fit_calc_wrapper(self.params) onlyfiles = [f for f in os.listdir(SML_TEST_LEGACY_ORBITAL_DIR) if os.path.isfile(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f)) and f.endswith('.csv') and f.__contains__('_method1_')] count = 0 for i, f in enumerate(onlyfiles): ifg_data = np.genfromtxt(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f), delimiter=',') for k, j in enumerate([m.tmp_sampled_path for m in multi_paths]): ifg = Ifg(j) ifg.open() print(f) print(j) if os.path.basename(j).split('_ifg.')[0] == os.path.basename(f).split( '_orb_planar_1lks_method1_geo_')[1].split('.')[0]: count += 1 # all numbers equal np.testing.assert_array_almost_equal(ifg_data, ifg.phase_data, decimal=2) # means must also be equal assert np.nanmean(ifg_data) == pytest.approx(np.nanmean(ifg.phase_data), abs=1e-2) # number of nans must equal assert np.sum(np.isnan(ifg_data)) == np.sum(np.isnan(ifg.phase_data)) ifg.close() # ensure that we have expected number of matches assert count == len(self.ifg_paths)
def __run_once(self): ref_phs_file = Configuration.ref_phs_file(self.params) correct._copy_mlooked(self.params) multi_paths = self.params[cf.INTERFEROGRAM_FILES] ifg_paths = [p.tmp_sampled_path for p in multi_paths] ifgs = [Ifg(i) for i in ifg_paths] self.params[cf.REFX_FOUND], self.params[ cf.REFY_FOUND] = ref_pixel_calc_wrapper(self.params) correct._create_ifg_dict(self.params) ref_phase_est_wrapper(self.params) for i in ifgs: i.open() phase_prev = [i.phase_data for i in ifgs] # assert ref_ph_file present assert ref_phs_file.exists() time_written = os.stat(ref_phs_file).st_mtime for i in ifgs: i.close() return phase_prev, time_written