def __workflow(params, tdir): copytree(params[cf.OBS_DIR], tdir) # manipulate params params[cf.OBS_DIR] = tdir.as_posix() outdir = tdir.joinpath('out') outdir.mkdir(exist_ok=True) params[cf.OUT_DIR] = outdir.as_posix() params[cf.DEM_FILE] = tdir.joinpath(Path( params[cf.DEM_FILE]).name).as_posix() params[cf.DEM_HEADER_FILE] = tdir.joinpath( Path(params[cf.DEM_HEADER_FILE]).name).as_posix() params[cf.HDR_FILE_LIST] = tdir.joinpath( Path(params[cf.HDR_FILE_LIST]).name).as_posix() params[cf.SLC_DIR] = tdir.as_posix() params[cf.IFG_FILE_LIST] = tdir.joinpath( Path(params[cf.IFG_FILE_LIST]).name).as_posix() params[cf.COH_FILE_DIR] = tdir.as_posix() params[cf.APS_INCIDENCE_MAP] = tdir.joinpath( Path(params[cf.APS_INCIDENCE_MAP]).name).as_posix() params[cf.TMPDIR] = tdir.joinpath(Path(params[cf.TMPDIR]).name).as_posix() output_conf = tdir.joinpath('roipac_temp.conf') cf.write_config_file(params=params, output_conf_file=output_conf) params = configuration.Configuration(output_conf).__dict__ conv2tif.main(params) prepifg.main(params) return params
def test_conv2tif_file_types(tempdir, gamma_conf): tdir = Path(tempdir()) params = manipulate_test_conf(gamma_conf, tdir) params[cf.COH_MASK] = 1 output_conf_file = 'conf.conf' output_conf = tdir.joinpath(output_conf_file) cf.write_config_file(params=params, output_conf_file=output_conf) params_s = configuration.Configuration(output_conf).__dict__ conv2tif.main(params_s) ifg_files = list( Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_ifg.tif')) coh_files = list( Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_coh.tif')) dem_file = list( Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_dem.tif'))[0] # assert coherence and ifgs have correct metadata for i in itertools.chain(*[ifg_files, coh_files]): ifg = Ifg(i) ifg.open() md = ifg.meta_data if i.name.endswith('_ifg.tif'): assert md[ifc.DATA_TYPE] == ifc.ORIG continue if i.name.endswith('_coh.tif'): assert md[ifc.DATA_TYPE] == ifc.COH continue # assert dem has correct metadata dem = DEM(dem_file.as_posix()) dem.open() md = dem.dataset.GetMetadata() assert md[ifc.DATA_TYPE] == ifc.DEM shutil.rmtree(tdir)
def modify_params(conf_file, parallel, output_conf_file): tdir = Path(tempdir()) params = manipulate_test_conf(conf_file, tdir) params[cf.COH_MASK] = coh_mask params[cf.LARGE_TIFS] = 1 params[cf.PARALLEL] = parallel params[cf.PROCESSES] = 4 params[cf.APSEST] = 1 params[cf.IFG_LKSX], params[cf.IFG_LKSY] = get_lks, get_lks params[cf.REFNX], params[cf.REFNY] = 4, 4 params[cf.IFG_CROP_OPT] = local_crop params[cf.ORBITAL_FIT_LOOKS_X], params[ cf.ORBITAL_FIT_LOOKS_Y] = orbfit_lks, orbfit_lks params[cf.ORBITAL_FIT] = 1 params[cf.ORBITAL_FIT_METHOD] = orbfit_method params[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees params[cf.REF_EST_METHOD] = ref_est_method params["rows"], params["cols"] = 3, 2 params["tiles"] = params["rows"] * params["cols"] print(params) # write new temp config output_conf = tdir.joinpath(output_conf_file) cf.write_config_file(params=params, output_conf_file=output_conf) return output_conf, params
def modify_params(conf_file, parallel_vs_serial, output_conf_file): tdir = Path(tempdir()) params = manipulate_test_conf(conf_file, tdir) if params[cf.PROCESSOR] == 1: # turn on coherence for gamma params[cf.COH_MASK] = 1 params[cf.PARALLEL] = parallel_vs_serial params[cf.PROCESSES] = 4 params[cf.APSEST] = 1 params[cf.IFG_LKSX], params[cf.IFG_LKSY] = get_lks, get_lks params[cf.REFNX], params[cf.REFNY] = 2, 2 params[cf.IFG_CROP_OPT] = get_crop params[cf.ORBITAL_FIT_LOOKS_X], params[ cf.ORBITAL_FIT_LOOKS_Y] = orbfit_lks, orbfit_lks params[cf.ORBITAL_FIT] = 1 params[cf.ORBITAL_FIT_METHOD] = orbfit_method params[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees params[cf.REF_EST_METHOD] = ref_est_method params["rows"], params["cols"] = 3, 2 params["savenpy"] = 1 params["tiles"] = params["rows"] * params["cols"] print(params) # write new temp config output_conf = tdir.joinpath(output_conf_file) cf.write_config_file(params=params, output_conf_file=output_conf) return output_conf, params
def test_vcm_legacy_vs_mpi(mpisync, tempdir, roipac_or_gamma_conf): params = configuration.Configuration(roipac_or_gamma_conf).__dict__ LEGACY_VCM_DIR = os.path.join(SML_TEST_DIR, 'vcm') legacy_vcm = np.genfromtxt(os.path.join(LEGACY_VCM_DIR, 'vcmt.csv'), delimiter=',') tmpdir = Path(mpiops.run_once(tempdir)) mpiops.run_once(common.copytree, params[cf.OBS_DIR], tmpdir) params[cf.OUT_DIR] = tmpdir.joinpath('out') params[cf.PARALLEL] = 0 output_conf = Path(tmpdir).joinpath('conf.cfg') cf.write_config_file(params=params, output_conf_file=output_conf) params = configuration.Configuration(output_conf).__dict__ dest_paths = [p.sampled_path for p in params[cf.INTERFEROGRAM_FILES]] # run conv2tif and prepifg, create the dest_paths files conv2tif.main(params) params[cf.INTERFEROGRAM_FILES].pop() prepifg.main(params) params[cf.INTERFEROGRAM_FILES].pop() preread_ifgs = process._create_ifg_dict(dest_paths, params=params) refpx, refpy = process._ref_pixel_calc(dest_paths, params) process._orb_fit_calc(params[cf.INTERFEROGRAM_FILES], params) process._ref_phase_estimation(dest_paths, params, refpx, refpy) maxvar, vcmt = process._maxvar_vcm_calc(dest_paths, params, preread_ifgs) # phase data after ref pixel has changed due to commit bf2f7ebd # Legacy tests won't match anymore np.testing.assert_array_almost_equal(maxvar, legacy_maxvar, decimal=4) np.testing.assert_array_almost_equal(legacy_vcm, vcmt, decimal=3) mpiops.run_once(shutil.rmtree, tmpdir)
def series_ifgs(gamma_conf): print('======================setup series==========================') tdir = Path(tempfile.mkdtemp()) params_s = manipulate_test_conf(gamma_conf, tdir) params_s[cf.PARALLEL] = 0 output_conf_file = 'conf.conf' output_conf = tdir.joinpath(output_conf_file) cf.write_config_file(params=params_s, output_conf_file=output_conf) params_s = Configuration(output_conf).__dict__ gtif_paths = conv2tif.main(params_s) prepifg.main(params_s) parallel_df = list(Path(tdir).joinpath('out').glob(glob_prefix)) s_ifgs = small_data_setup(datafiles=parallel_df) yield s_ifgs print('======================teardown series==========================') shutil.rmtree(params_s[cf.OBS_DIR])
def test_new_config_file_and_original_match(self): params = config.get_config_params(TEST_CONF_GAMMA) temp_config = tempfile.mktemp(suffix='.conf') config.write_config_file(params, temp_config) new_params = config.get_config_params(temp_config) self.maxDiff = None self.assertDictEqual(params, new_params) os.remove(temp_config)
def test_coherence_files_exist(self): self.params[COH_MASK] = 1 self.params[COH_FILE_LIST] = os.path.join(common.SML_TEST_GAMMA, 'ifms_17') validate_coherence_files(self.params[IFG_FILE_LIST], self.params) self.params[COH_FILE_LIST] = os.path.join(common.SML_TEST_GAMMA, 'bad_epochs_ifms_17') tmp_config = tempfile.mktemp('.conf') write_config_file(self.params, output_conf_file=tmp_config) with pytest.raises(ConfigException): Configuration(tmp_config).__dict__
def create_stack_output(tempdir, gamma_conf): tdir = Path(tempdir()) params = manipulate_test_conf(gamma_conf, tdir) output_conf_file = tdir.joinpath('conf.cfg') output_conf = tdir.joinpath(output_conf_file) cf.write_config_file(params=params, output_conf_file=output_conf) check_call(f"pyrate conv2tif -f {output_conf}", shell=True) check_call(f"pyrate prepifg -f {output_conf}", shell=True) check_call(f"pyrate process -f {output_conf}", shell=True) params = Configuration(output_conf).__dict__ return params, tdir
def _get_mlooked_files(gamma_conf, tdir, refx, refy): params = manipulate_test_conf(gamma_conf, tdir) params[cf.REFX] = refx params[cf.REFY] = refy output_conf_file = 'config.conf' output_conf = tdir.joinpath(output_conf_file) cf.write_config_file(params=params, output_conf_file=output_conf) check_call(f"pyrate conv2tif -f {output_conf}", shell=True) check_call(f"pyrate prepifg -f {output_conf}", shell=True) stdout = run(f"pyrate process -f {output_conf}", shell=True, capture_output=True, text=True) print("============================================", stdout) return params, stdout
def setUp(self): self.params = cf.get_config_params(TEST_CONF_ROIPAC) self.params[cf.PARALLEL] = 0 self.params[cf.OUT_DIR], self.ifg_paths = copy_small_ifg_file_list() conf_file = Path(self.params[cf.OUT_DIR], 'conf_file.conf') cf.write_config_file(params=self.params, output_conf_file=conf_file) self.params = Configuration(conf_file).__dict__ self.params_alt_ref_frac = copy.copy(self.params) self.params_alt_ref_frac[cf.REF_MIN_FRAC] = 0.5 self.params_all_2s = copy.copy(self.params) self.params_all_2s[cf.REFNX] = 2 self.params_all_2s[cf.REFNY] = 2 self.params_chipsize_15 = copy.copy(self.params_all_2s) self.params_chipsize_15[cf.REF_CHIP_SIZE] = 15 self.params_all_1s = copy.copy(self.params) self.params_all_1s[cf.REFNX] = 1 self.params_all_1s[cf.REFNY] = 1 self.params_all_1s[cf.REF_MIN_FRAC] = 0.7
def test_prepifg_treat_inputs_read_only(gamma_conf, tempdir, coh_mask): tdir = Path(tempdir()) params = common.manipulate_test_conf(gamma_conf, tdir) params[cf.COH_MASK] = coh_mask output_conf = tdir.joinpath('conf.cfg') cf.write_config_file(params=params, output_conf_file=output_conf) check_call(f"mpirun -n 3 pyrate conv2tif -f {output_conf}", shell=True) tifs = list(Path(params[cf.OUT_DIR]).glob('*_unw_ifg.tif')) assert len(tifs) == 17 check_call(f"mpirun -n 3 pyrate prepifg -f {output_conf}", shell=True) cropped = list(Path(params[cf.OUT_DIR]).glob('*cr.tif')) if coh_mask: # 17 + 1 dem + 17 coh files assert len(cropped) == 35 else: # 17 + 1 dem assert len(cropped) == 18 # check all tifs from conv2tif are still readonly for t in tifs: assert t.stat().st_mode == 33060
def parallel_ifgs(gamma_conf): tdir = Path(tempfile.mkdtemp()) params_p = manipulate_test_conf(gamma_conf, tdir) params_p[cf.PARALLEL] = 1 output_conf_file = 'conf.conf' output_conf = tdir.joinpath(output_conf_file) cf.write_config_file(params=params_p, output_conf_file=output_conf) params_p = Configuration(output_conf).__dict__ gtif_paths = conv2tif.main(params_p) prepifg.main(params_p) parallel_df = list(Path(tdir).joinpath('out').glob(glob_prefix)) p_ifgs = small_data_setup(datafiles=parallel_df) yield p_ifgs shutil.rmtree(params_p[cf.OBS_DIR])
def test_prepifg_file_types(tempdir, gamma_conf, coh_mask): tdir = Path(tempdir()) params = manipulate_test_conf(gamma_conf, tdir) params[cf.COH_MASK] = coh_mask params[cf.PARALLEL] = 0 output_conf_file = 'conf.conf' output_conf = tdir.joinpath(output_conf_file) cf.write_config_file(params=params, output_conf_file=output_conf) params_s = Configuration(output_conf).__dict__ conv2tif.main(params_s) # reread params from config params_s = Configuration(output_conf).__dict__ prepifg.main(params_s) ifg_files = list( Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_ifg.tif')) assert len(ifg_files) == 17 mlooked_files = list( Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_ifg_1rlks_1cr.tif')) assert len(mlooked_files) == 17 coh_files = list( Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_cc_coh.tif')) mlooked_coh_files = list( Path(tdir.joinpath( params_s[cf.OUT_DIR])).glob('*_cc_coh_1rlks_1cr.tif')) if coh_mask: assert len(coh_files) == 17 assert len(mlooked_coh_files) == 17 dem_file = list( Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_dem.tif'))[0] mlooked_dem_file = list( Path(tdir.joinpath( params_s[cf.OUT_DIR])).glob('*_dem_1rlks_1cr.tif'))[0] import itertools # assert coherence and ifgs have correct metadata for i in itertools.chain( *[ifg_files, mlooked_files, coh_files, mlooked_coh_files]): ifg = Ifg(i) ifg.open() md = ifg.meta_data if i.name.endswith('_ifg.tif'): assert md[ifc.DATA_TYPE] == ifc.ORIG continue if i.name.endswith('_coh.tif'): assert md[ifc.DATA_TYPE] == ifc.COH continue if i.name.endswith('_cc_coh_1rlks_1cr.tif'): assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED_COH continue if i.name.endswith('_ifg_1rlks_1cr.tif'): if coh_mask: assert md[ifc.DATA_TYPE] == ifc.COHERENCE else: assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED continue # assert dem has correct metadata dem = DEM(dem_file.as_posix()) dem.open() md = dem.dataset.GetMetadata() assert md[ifc.DATA_TYPE] == ifc.DEM dem = DEM(mlooked_dem_file.as_posix()) dem.open() md = dem.dataset.GetMetadata() assert md[ifc.DATA_TYPE] == ifc.MLOOKED_DEM shutil.rmtree(tdir)
def test_write_config_file(self): params = config.get_config_params(TEST_CONF_GAMMA) temp_config = tempfile.mktemp(suffix='.conf') config.write_config_file(params, temp_config) self.assertTrue(os.path.exists(temp_config)) os.remove(temp_config)