def setUpClass(cls): # read in the params _, _, params = cf.get_ifg_paths(TEST_CONF_GAMMA) glob_prefix = "*utm_unw_1rlks_1cr.tif" # SERIAL cls.serial_dir = tempfile.mkdtemp() params[cf.OUT_DIR] = cls.serial_dir params[cf.PARALLEL] = False shared.mkdir_p(cls.serial_dir) gtif_paths = conv2tif.main(params) prepifg.main(params) serial_df = glob.glob(os.path.join(cls.serial_dir, glob_prefix)) cls.serial_ifgs = small_data_setup(datafiles=serial_df) # Clean up serial converted tifs so we can test parallel conversion common.remove_tifs(SML_TEST_GAMMA) # PARALLEL cls.parallel_dir = tempfile.mkdtemp() params[cf.OUT_DIR] = cls.parallel_dir params[cf.PARALLEL] = True shared.mkdir_p(cls.parallel_dir) gtif_paths = conv2tif.main(params) prepifg.main(params) para_df = glob.glob(os.path.join(cls.parallel_dir, glob_prefix)) cls.para_ifgs = small_data_setup(datafiles=para_df)
def test_prepifg_mpi(mpisync, get_config, tempdir, roipac_or_gamma, get_lks, get_crop): from tests.common import TEST_CONF_ROIPAC, TEST_CONF_GAMMA from os.path import join, basename if roipac_or_gamma == 1: params = get_config(TEST_CONF_GAMMA) else: params = get_config(TEST_CONF_ROIPAC) outdir = mpiops.run_once(tempdir) params[cf.OUT_DIR] = outdir params[cf.PARALLEL] = False params[cf.IFG_LKSX], params[cf.IFG_LKSY] = get_lks, get_lks params[cf.IFG_CROP_OPT] = get_crop if roipac_or_gamma == 1: params[cf.IFG_FILE_LIST] = join(common.SML_TEST_GAMMA, 'ifms_17') params[cf.OBS_DIR] = common.SML_TEST_GAMMA params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA params[cf.DEM_HEADER_FILE] = common.SML_TEST_DEM_HDR_GAMMA conv2tif.main(params) prepifg.main(params) common.remove_tifs(params[cf.OBS_DIR]) if mpiops.rank == 0: if roipac_or_gamma == 1: params_s = get_config(TEST_CONF_GAMMA) else: params_s = get_config(TEST_CONF_ROIPAC) params_s[cf.OUT_DIR] = tempdir() params_s[cf.PARALLEL] = True params_s[cf.IFG_LKSX], params_s[cf.IFG_LKSY] = get_lks, get_lks params_s[cf.IFG_CROP_OPT] = get_crop conv2tif.main(params) if roipac_or_gamma == 1: base_unw_paths = glob.glob(join(common.SML_TEST_GAMMA, "*_utm.unw")) prepifg.main(params) else: base_unw_paths = glob.glob(join(common.SML_TEST_OBS, "*.unw")) prepifg.main(params_s) mpi_tifs = glob.glob(join(outdir, "*.tif")) serial_tifs = glob.glob(join(params[cf.OUT_DIR], "*.tif")) mpi_tifs.sort() serial_tifs.sort() # 17 geotifs, and 17 mlooked tifs assert len(mpi_tifs) == len(serial_tifs) for m_f, s_f in zip(mpi_tifs, serial_tifs): assert basename(m_f) == basename(s_f) shutil.rmtree(outdir) shutil.rmtree(params_s[cf.OUT_DIR]) common.remove_tifs(params[cf.OBS_DIR])
def tearDownClass(cls): try: shutil.rmtree(cls.temp_out_dir) except PermissionError: print("File opened by another process.") try: common.remove_tifs( cf.get_config_params(common.TEST_CONF_ROIPAC)[cf.OBS_DIR]) except PermissionError: print("File opened by another process.") for ifg in cls.ifgs: ifg.close()
def tearDownClass(cls): try: shutil.rmtree(cls.parallel_dir) except PermissionError: print("File opened by another process.") try: shutil.rmtree(cls.serial_dir) except PermissionError: print("File opened by another process.") try: common.remove_tifs(SML_TEST_GAMMA) except PermissionError: print("File opened by another process.")
def test_vcm_legacy_vs_mpi(mpisync, tempdir, get_config): from tests.common import SML_TEST_DIR, TEST_CONF_ROIPAC params_dict = get_config(TEST_CONF_ROIPAC) LEGACY_VCM_DIR = os.path.join(SML_TEST_DIR, 'vcm') legacy_vcm = np.genfromtxt(os.path.join(LEGACY_VCM_DIR, 'vcmt.csv'), delimiter=',') if mpiops.rank == 0: outdir = tempdir() else: outdir = None outdir = mpiops.comm.bcast(outdir, root=0) params_dict[cf.OUT_DIR] = outdir params_dict[cf.PARALLEL] = False xlks, ylks, crop = cf.transform_params(params_dict) base_unw_paths = cf.original_ifg_paths(params_dict[cf.IFG_FILE_LIST], params_dict[cf.OBS_DIR]) # dest_paths are tifs that have been geotif converted and multilooked dest_paths = cf.get_dest_paths(base_unw_paths, crop, params_dict, xlks) # run prepifg, create the dest_paths files if mpiops.rank == 0: conv2tif.main(params_dict) prepifg.main(params_dict) mpiops.comm.barrier() tiles = pyrate.core.shared.get_tiles(dest_paths[0], rows=1, cols=1) preread_ifgs = process._create_ifg_dict(dest_paths, params=params_dict, tiles=tiles) refpx, refpy = process._ref_pixel_calc(dest_paths, params_dict) process._orb_fit_calc(dest_paths, params_dict) process._ref_phase_estimation(dest_paths, params_dict, refpx, refpy) maxvar, vcmt = process._maxvar_vcm_calc(dest_paths, params_dict, preread_ifgs) np.testing.assert_array_almost_equal(maxvar, legacy_maxvar, decimal=4) np.testing.assert_array_almost_equal(legacy_vcm, vcmt, decimal=3) if mpiops.rank == 0: shutil.rmtree(outdir) common.remove_tifs(params_dict[cf.OBS_DIR])
def tearDownClass(cls): for ifg in cls.ifgs: ifg.close() shutil.rmtree(cls.temp_out_dir) common.remove_tifs( cf.get_config_params(common.TEST_CONF_ROIPAC)[cf.OBS_DIR])
def tearDownClass(cls): for i in cls.ifgs: i.close() shutil.rmtree(cls.tif_dir) common.remove_tifs(cls.params[cf.OBS_DIR])
def teardown_method(self, method): common.remove_tifs(self.gamma_params[cf.OBS_DIR]) common.remove_tifs(self.roipac_params[cf.OBS_DIR])
def setUpClass(cls): rate_types = ['linrate', 'linerror', 'linsamples'] cls.tif_dir = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_GAMMA # change the required params params = cf.get_config_params(cls.test_conf) params[cf.OBS_DIR] = common.SML_TEST_GAMMA params[cf.PROCESSOR] = 1 # gamma params[cf.IFG_FILE_LIST] = os.path.join( common.SML_TEST_GAMMA, 'ifms_17') params[cf.OUT_DIR] = cls.tif_dir params[cf.PARALLEL] = 1 params[cf.APS_CORRECTION] = False params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) xlks, ylks, crop = cf.transform_params(params) # base_unw_paths need to be geotiffed by converttogeotif # and multilooked by run_prepifg base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST], params[cf.OBS_DIR]) # dest_paths are tifs that have been geotif converted and multilooked cls.dest_paths = cf.get_dest_paths( base_unw_paths, crop, params, xlks) gtif_paths = conv2tif.do_geotiff(base_unw_paths, params) prepifg.do_prepifg(gtif_paths, params) tiles = pyrate.core.shared.get_tiles(cls.dest_paths[0], 3, 3) ifgs = common.small_data_setup() cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = \ process.process_ifgs(cls.dest_paths, params, 3, 3) cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) cls.rate_p, cls.error_p, cls.samples_p = [ common.reconstruct_linrate( ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types ] common.remove_tifs(params[cf.OBS_DIR]) # now create the non parallel version cls.tif_dir_s = tempfile.mkdtemp() params[cf.PARALLEL] = 0 params[cf.OUT_DIR] = cls.tif_dir_s params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) cls.dest_paths_s = cf.get_dest_paths( base_unw_paths, crop, params, xlks) gtif_paths = conv2tif.do_geotiff(base_unw_paths, params) prepifg.do_prepifg(gtif_paths, params) cls.refpixel, cls.maxvar, cls.vcmt = \ process.process_ifgs(cls.dest_paths_s, params, 3, 3) cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) cls.rate, cls.error, cls.samples = [ common.reconstruct_linrate( ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types ]
def tearDownClass(cls): shutil.rmtree(cls.tif_dir, ignore_errors=True) shutil.rmtree(cls.tif_dir_s, ignore_errors=True) common.remove_tifs(cf.get_config_params(cls.test_conf)[cf.OBS_DIR])
def teardown_class(cls): params = Configuration(cls.conf_file).__dict__ shutil.rmtree(cls.base_dir) common.remove_tifs(params[WORKING_DIR])
def tearDownClass(cls): shutil.rmtree(cls.gamma_base_dir) shutil.rmtree(cls.roipac_base_dir) remove_tifs(cls.SMLNEY_GAMMA_TEST) remove_tifs(common.SML_TEST_OBS)
def teardown_method(self, method): common.remove_tifs(self.params[cf.OBS_DIR])
def tearDown(self): params = cf.get_config_params(self.conf_file) shutil.rmtree(self.base_dir) common.remove_tifs(params[cf.OBS_DIR])
def tearDownClass(cls): shutil.rmtree(cls.temp_out_dir) params = cf.get_config_params(TEST_CONF_ROIPAC) remove_tifs(params[cf.OBS_DIR])
def setup_class(cls): gamma_conf = common.TEST_CONF_GAMMA from tests.common import manipulate_test_conf rate_types = ['stack_rate', 'stack_error', 'stack_samples'] cls.tif_dir = Path(tempfile.mkdtemp()) params = manipulate_test_conf(gamma_conf, cls.tif_dir) from pyrate.configuration import Configuration # change the required params params[C.PROCESSES] = 4 params[C.PROCESSOR] = 1 # gamma params[C.IFG_FILE_LIST] = os.path.join(common.GAMMA_SML_TEST_DIR, 'ifms_17') params[C.PARALLEL] = 1 params[C.APS_CORRECTION] = 0 params[C.REFX], params[C.REFY] = -1, -1 rows, cols = params["rows"], params["cols"] output_conf_file = 'gamma.conf' output_conf = cls.tif_dir.joinpath(output_conf_file).as_posix() pyrate.configuration.write_config_file(params=params, output_conf_file=output_conf) config = Configuration(output_conf) params = config.__dict__ common.sub_process_run(f"pyrate conv2tif -f {output_conf}") common.sub_process_run(f"pyrate prepifg -f {output_conf}") cls.sampled_paths = [ p.tmp_sampled_path for p in params[C.INTERFEROGRAM_FILES] ] ifgs = common.small_data_setup() correct._copy_mlooked(params) tiles = pyrate.core.shared.get_tiles(cls.sampled_paths[0], rows, cols) correct.correct_ifgs(config) pyrate.main.timeseries(config) pyrate.main.stack(config) cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = \ (params[C.REFX], params[C.REFY]), params[C.MAXVAR], params[ C.VCMT] cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles, params[C.OUT_DIR]) cls.rate_p, cls.error_p, cls.samples_p = \ [common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[C.TMPDIR], t) for t in rate_types] common.remove_tifs(params[C.OUT_DIR]) # now create the non parallel version cls.tif_dir_s = Path(tempfile.mkdtemp()) params = manipulate_test_conf(gamma_conf, cls.tif_dir_s) params[C.PROCESSES] = 4 params[C.PROCESSOR] = 1 # gamma params[C.IFG_FILE_LIST] = os.path.join(common.GAMMA_SML_TEST_DIR, 'ifms_17') params[C.PARALLEL] = 0 params[C.APS_CORRECTION] = 0 params[C.REFX], params[C.REFY] = -1, -1 output_conf_file = 'gamma.conf' output_conf = cls.tif_dir_s.joinpath(output_conf_file).as_posix() pyrate.configuration.write_config_file(params=params, output_conf_file=output_conf) config = Configuration(output_conf) params = config.__dict__ common.sub_process_run(f"pyrate conv2tif -f {output_conf}") common.sub_process_run(f"pyrate prepifg -f {output_conf}") correct._copy_mlooked(params) correct.correct_ifgs(config) pyrate.main.timeseries(config) pyrate.main.stack(config) cls.refpixel, cls.maxvar, cls.vcmt = \ (params[C.REFX], params[C.REFY]), params[C.MAXVAR], params[ C.VCMT] cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles, params[C.OUT_DIR]) cls.rate, cls.error, cls.samples = \ [common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[C.TMPDIR], t) for t in rate_types] cls.params = params
def tearDownClass(cls): common.remove_tifs(cls.params[OBS_DIR]) common.remove_tifs(cls.params[OUT_DIR])
def teardown_class(cls): shutil.rmtree(cls.temp_out_dir) common.remove_tifs( cf.get_config_params(common.TEST_CONF_ROIPAC)[cf.OBS_DIR])
def setUpClass(cls): rate_types = ['stack_rate', 'stack_error', 'stack_samples'] cls.tif_dir = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_GAMMA from pyrate.configuration import Configuration # change the required params params = Configuration(cls.test_conf).__dict__ params[cf.OBS_DIR] = common.SML_TEST_GAMMA params[cf.PROCESSES] = 4 params[cf.PROCESSOR] = 1 # gamma params[cf.IFG_FILE_LIST] = os.path.join(common.SML_TEST_GAMMA, 'ifms_17') params[cf.OUT_DIR] = cls.tif_dir params[cf.PARALLEL] = 1 params[cf.APS_CORRECTION] = False params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) rows, cols = params["rows"], params["cols"] # xlks, ylks, crop = cf.transform_params(params) # base_unw_paths need to be geotiffed by converttogeotif # and multilooked by run_prepifg base_unw_paths = list(cf.parse_namelist(params[cf.IFG_FILE_LIST])) multi_paths = [ MultiplePaths(params[cf.OUT_DIR], b, ifglksx=params[cf.IFG_LKSX], ifgcropopt=params[cf.IFG_CROP_OPT]) for b in base_unw_paths ] # dest_paths are tifs that have been geotif converted and multilooked cls.converted_paths = [b.converted_path for b in multi_paths] cls.sampled_paths = [b.sampled_path for b in multi_paths] from copy import copy orig_params = copy(params) conv2tif.main(params) prepifg.main(orig_params) tiles = pyrate.core.shared.get_tiles(cls.sampled_paths[0], rows, cols) ifgs = common.small_data_setup() params[cf.INTERFEROGRAM_FILES] = multi_paths cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = process.process_ifgs( cls.sampled_paths, params, rows, cols) cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) cls.rate_p, cls.error_p, cls.samples_p = \ [common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types] common.remove_tifs(params[cf.OBS_DIR]) # now create the non parallel version cls.tif_dir_s = tempfile.mkdtemp() params[cf.PARALLEL] = 0 params[cf.PROCESSES] = 1 params[cf.OUT_DIR] = cls.tif_dir_s params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) multi_paths = [ MultiplePaths(params[cf.OUT_DIR], b, ifglksx=params[cf.IFG_LKSX], ifgcropopt=params[cf.IFG_CROP_OPT]) for b in base_unw_paths ] cls.converted_paths_s = [b.converted_path for b in multi_paths] cls.sampled_paths_s = [b.sampled_path for b in multi_paths] orig_params = copy(params) conv2tif.main(params) prepifg.main(orig_params) params[cf.INTERFEROGRAM_FILES] = multi_paths cls.refpixel, cls.maxvar, cls.vcmt = process.process_ifgs( cls.sampled_paths_s, params, rows, cols) cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) cls.rate, cls.error, cls.samples = \ [common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types]