def setUpClass(cls): cls.tif_dir = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_GAMMA # change the required params cls.params = cf.get_config_params(cls.test_conf) cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA cls.params[cf.PROCESSOR] = 1 # gamma file_list = list(cf.parse_namelist(os.path.join(common.SML_TEST_GAMMA, 'ifms_17'))) fd, cls.params[cf.IFG_FILE_LIST] = tempfile.mkstemp(suffix='.conf', dir=cls.tif_dir) os.close(fd) # write a short filelist with only 3 gamma unws with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp: for f in file_list[:3]: fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n') cls.params[cf.OUT_DIR] = cls.tif_dir cls.params[cf.PARALLEL] = 0 cls.params[cf.REF_EST_METHOD] = 1 cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA # base_unw_paths need to be geotiffed and multilooked by run_prepifg cls.base_unw_paths = cf.original_ifg_paths( cls.params[cf.IFG_FILE_LIST]) cls.base_unw_paths.append(common.SML_TEST_DEM_GAMMA) xlks, ylks, crop = cf.transform_params(cls.params) # dest_paths are tifs that have been geotif converted and multilooked run_prepifg.gamma_prepifg(cls.base_unw_paths, cls.params) cls.base_unw_paths.pop() # removed dem as we don't want it in ifgs dest_paths = cf.get_dest_paths( cls.base_unw_paths, crop, cls.params, xlks) cls.ifgs = common.small_data_setup(datafiles=dest_paths)
def setUpClass(cls): cls.tif_dir = tempfile.mkdtemp() cls.tif_dir_method2 = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_ROIPAC # change the required params cls.params = cf.get_config_params(cls.test_conf) cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA cls.params[cf.PROCESSOR] = 1 # gamma file_list = cf.parse_namelist( os.path.join(common.SML_TEST_GAMMA, 'ifms_17')) cls.params[cf.IFG_FILE_LIST] = tempfile.mktemp(dir=cls.tif_dir) # write a short filelist with only 3 gamma unws with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp: for f in file_list[:2]: fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n') cls.params[cf.OUT_DIR] = cls.tif_dir cls.params[cf.PARALLEL] = True cls.params[cf.REF_EST_METHOD] = 1 cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA cls.params[cf.APS_INCIDENCE_MAP] = common.SML_TEST_INCIDENCE # base_unw_paths need to be geotiffed and multilooked by run_prepifg base_unw_paths = run_pyrate.original_ifg_paths( cls.params[cf.IFG_FILE_LIST]) # add dem base_unw_paths.append(common.SML_TEST_DEM_GAMMA) # add incidence base_unw_paths.append(common.SML_TEST_INCIDENCE) xlks, ylks, crop = run_pyrate.transform_params(cls.params) import copy cls.params_method2 = copy.copy(cls.params) cls.params_method2[cf.OUT_DIR] = cls.tif_dir_method2 cls.params_method2[cf.APS_METHOD] = 2 # dest_paths are tifs that have been geotif converted and multilooked run_prepifg.gamma_prepifg(base_unw_paths, cls.params) run_prepifg.gamma_prepifg(base_unw_paths, cls.params_method2) # removed incidence as we don't want it in ifgs list base_unw_paths.pop() # removed dem as we don't want it in ifgs list base_unw_paths.pop() dest_paths = run_pyrate.get_dest_paths(base_unw_paths, crop, cls.params, xlks) cls.ifgs = common.small_data_setup(datafiles=dest_paths) dest_paths_m2 = run_pyrate.get_dest_paths(base_unw_paths, crop, cls.params_method2, xlks) cls.ifgs_method2 = common.small_data_setup(datafiles=dest_paths_m2) aps.remove_aps_delay(cls.ifgs, cls.params) aps.remove_aps_delay(cls.ifgs_method2, cls.params_method2)
def setUpClass(cls): rate_types = ['linrate', 'linerror', 'linsamples'] cls.tif_dir = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_GAMMA # change the required params params = cf.get_config_params(cls.test_conf) params[cf.OBS_DIR] = common.SML_TEST_GAMMA params[cf.PROCESSOR] = 1 # gamma params[cf.IFG_FILE_LIST] = os.path.join(common.SML_TEST_GAMMA, 'ifms_17') params[cf.OUT_DIR] = cls.tif_dir params[cf.PARALLEL] = 0 params[cf.APS_CORRECTION] = False params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) xlks, ylks, crop = cf.transform_params(params) # base_unw_paths need to be geotiffed and multilooked by run_prepifg base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) # dest_paths are tifs that have been geotif converted and multilooked cls.dest_paths = cf.get_dest_paths(base_unw_paths, crop, params, xlks) run_prepifg.gamma_prepifg(base_unw_paths, params) tiles = run_pyrate.get_tiles(cls.dest_paths[0], 3, 3) ifgs = common.small_data_setup() cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = \ run_pyrate.process_ifgs(cls.dest_paths, params, 3, 3) cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) cls.rate_p, cls.error_p, cls.samples_p = [ common.reconstruct_linrate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types ] # now create the non parallel version cls.tif_dir_s = tempfile.mkdtemp() params[cf.PARALLEL] = 0 params[cf.OUT_DIR] = cls.tif_dir_s params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) cls.dest_paths_s = cf.get_dest_paths(base_unw_paths, crop, params, xlks) run_prepifg.gamma_prepifg(base_unw_paths, params) cls.refpixel, cls.maxvar, cls.vcmt = \ run_pyrate.process_ifgs(cls.dest_paths_s, params, 3, 3) cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) cls.rate, cls.error, cls.samples = [ common.reconstruct_linrate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types ]
def test_prepifg_mpi(mpisync, get_config, tempdir, roipac_or_gamma, get_lks, get_crop): from tests.common import TEST_CONF_ROIPAC, TEST_CONF_GAMMA from os.path import join, basename if roipac_or_gamma == 1: params = get_config(TEST_CONF_GAMMA) else: params = get_config(TEST_CONF_ROIPAC) outdir = mpiops.run_once(tempdir) params[cf.OUT_DIR] = outdir params[cf.PARALLEL] = False params[cf.IFG_LKSX], params[cf.IFG_LKSY] = get_lks, get_lks params[cf.IFG_CROP_OPT] = get_crop if roipac_or_gamma == 1: params[cf.IFG_FILE_LIST] = join(common.SML_TEST_GAMMA, 'ifms_17') params[cf.OBS_DIR] = common.SML_TEST_GAMMA params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA params[cf.DEM_HEADER_FILE] = common.SML_TEST_DEM_HDR_GAMMA run_prepifg.main(params) if mpiops.rank == 0: if roipac_or_gamma == 1: params_s = get_config(TEST_CONF_GAMMA) else: params_s = get_config(TEST_CONF_ROIPAC) params_s[cf.OUT_DIR] = tempdir() params_s[cf.PARALLEL] = True params_s[cf.IFG_LKSX], params_s[cf.IFG_LKSY] = get_lks, get_lks params_s[cf.IFG_CROP_OPT] = get_crop if roipac_or_gamma == 1: base_unw_paths = glob.glob(join(common.SML_TEST_GAMMA, "*_utm.unw")) run_prepifg.gamma_prepifg(base_unw_paths, params_s) else: base_unw_paths = glob.glob(join(common.SML_TEST_OBS, "*.unw")) run_prepifg.roipac_prepifg(base_unw_paths, params_s) mpi_tifs = glob.glob(join(outdir, "*.tif")) serial_tifs = glob.glob(join(params[cf.OUT_DIR], "*.tif")) mpi_tifs.sort() serial_tifs.sort() # 17 geotifs, and 17 mlooked tifs assert len(mpi_tifs) == len(serial_tifs) for m_f, s_f in zip(mpi_tifs, serial_tifs): assert basename(m_f) == basename(s_f) shutil.rmtree(outdir) shutil.rmtree(params_s[cf.OUT_DIR])
def setUpClass(cls): cls.serial_dir = tempfile.mkdtemp() cls.parallel_dir = tempfile.mkdtemp() unw_paths = glob.glob(os.path.join(SML_TEST_GAMMA, "*_utm.unw")) # read in the params _, _, params = cf.get_ifg_paths(TEST_CONF_GAMMA) params[cf.OUT_DIR] = cls.serial_dir params[cf.PARALLEL] = False shared.mkdir_p(cls.serial_dir) run_prepifg.gamma_prepifg(unw_paths, params) params[cf.OUT_DIR] = cls.parallel_dir params[cf.PARALLEL] = True shared.mkdir_p(cls.parallel_dir) run_prepifg.gamma_prepifg(unw_paths, params)
def test_timeseries_linrate_mpi(mpisync, tempdir, modify_config, ref_est_method, row_splits, col_splits, get_crop, orbfit_lks, orbfit_method, orbfit_degrees): params = modify_config outdir = mpiops.run_once(tempdir) params[cf.OUT_DIR] = outdir params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) params[cf.DEM_HEADER_FILE] = SML_TEST_DEM_HDR_GAMMA params[cf.REF_EST_METHOD] = ref_est_method params[cf.IFG_CROP_OPT] = get_crop params[cf.ORBITAL_FIT_LOOKS_Y] = orbfit_lks params[cf.ORBITAL_FIT_LOOKS_X] = orbfit_lks params[cf.ORBITAL_FIT_METHOD] = orbfit_method params[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees xlks, ylks, crop = cf.transform_params(params) if xlks * col_splits > 45 or ylks * row_splits > 70: print('skipping test because lks and col_splits are not compatible') return # skip some tests in travis to run CI faster if TRAVIS and (xlks % 2 or row_splits % 2 or col_splits % 2 or orbfit_lks % 2): print('Skipping in travis env for faster CI run') return print("xlks={}, ref_est_method={}, row_splits={}, col_splits={}, " "get_crop={}, orbfit_lks={}, orbfit_method={}, " "rank={}".format(xlks, ref_est_method, row_splits, col_splits, get_crop, orbfit_lks, orbfit_method, orbfit_degrees, mpiops.rank)) base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) # dest_paths are tifs that have been geotif converted and multilooked dest_paths = cf.get_dest_paths(base_unw_paths, crop, params, xlks) # run prepifg, create the dest_paths files if mpiops.rank == 0: run_prepifg.gamma_prepifg(base_unw_paths, params) mpiops.comm.barrier() (refpx, refpy), maxvar, vcmt = run_pyrate.process_ifgs(ifg_paths=dest_paths, params=params, rows=row_splits, cols=col_splits) tiles = mpiops.run_once(run_pyrate.get_tiles, dest_paths[0], rows=row_splits, cols=col_splits) postprocessing.postprocess_linrate(row_splits, col_splits, params) postprocessing.postprocess_timeseries(row_splits, col_splits, params) ifgs_mpi_out_dir = params[cf.OUT_DIR] ifgs_mpi = small_data_setup(datafiles=dest_paths) # single process timeseries/linrate calculation if mpiops.rank == 0: params_old = modify_config params_old[cf.OUT_DIR] = tempdir() params_old[cf.REF_EST_METHOD] = ref_est_method params_old[cf.IFG_CROP_OPT] = get_crop params_old[cf.ORBITAL_FIT_LOOKS_Y] = orbfit_lks params_old[cf.ORBITAL_FIT_LOOKS_X] = orbfit_lks params_old[cf.ORBITAL_FIT_METHOD] = orbfit_method params_old[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees xlks, ylks, crop = cf.transform_params(params_old) base_unw_paths = cf.original_ifg_paths(params_old[cf.IFG_FILE_LIST]) dest_paths = cf.get_dest_paths(base_unw_paths, crop, params_old, xlks) run_prepifg.gamma_prepifg(base_unw_paths, params_old) ifgs = shared.pre_prepare_ifgs(dest_paths, params_old) mst_grid = tests.common.mst_calculation(dest_paths, params_old) refy, refx = refpixel.ref_pixel(ifgs, params_old) assert (refx == refpx) and (refy == refpy) # both must match pyrate.orbital.remove_orbital_error(ifgs, params_old) ifgs = common.prepare_ifgs_without_phase(dest_paths, params_old) rpe.estimate_ref_phase(ifgs, params_old, refx, refy) ifgs = shared.pre_prepare_ifgs(dest_paths, params_old) maxvar_s = [vcm.cvd(i, params_old)[0] for i in ifgs] vcmt_s = vcm.get_vcmt(ifgs, maxvar) tsincr, tscum, _ = tests.common.compute_time_series( ifgs, mst_grid, params, vcmt) rate, error, samples = tests.common.calculate_linear_rate( ifgs, params_old, vcmt, mst_grid) mst_mpi = reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) np.testing.assert_array_almost_equal(mst_grid, mst_mpi) tsincr_mpi, tscum_mpi = reconstruct_times_series( ifgs[0].shape, tiles, params[cf.TMPDIR]) rate_mpi, error_mpi, samples_mpi = \ [reconstruct_linrate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in ['linrate', 'linerror', 'linsamples']] np.testing.assert_array_almost_equal(maxvar, maxvar_s) np.testing.assert_array_almost_equal(vcmt, vcmt_s) for i, j in zip(ifgs, ifgs_mpi): np.testing.assert_array_almost_equal(i.phase_data, j.phase_data) np.testing.assert_array_almost_equal(tsincr, tsincr_mpi, decimal=4) np.testing.assert_array_almost_equal(tscum, tscum_mpi, decimal=4) np.testing.assert_array_almost_equal(rate, rate_mpi, decimal=4) np.testing.assert_array_almost_equal(error, error_mpi, decimal=4) np.testing.assert_array_almost_equal(samples, samples_mpi, decimal=4) # assert linear rate output tifs are same _tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'linrate.tif') _tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'linerror.tif') _tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'linsamples.tif') # assert time series output tifs are same epochlist = algorithm.get_epochs(ifgs)[0] for i in range(tsincr.shape[2]): _tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'tsincr' + '_' + str(epochlist.dates[i + 1]) + ".tif") # 12 timeseries outputs assert i + 1 == tsincr.shape[2] shutil.rmtree(ifgs_mpi_out_dir) # remove mpi out dir shutil.rmtree(params_old[cf.OUT_DIR]) # remove serial out dir
def setUpClass(cls): cls.tif_dir_serial = tempfile.mkdtemp() cls.tif_dir_mpi = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_ROIPAC # change the required params cls.params = cf.get_config_params(cls.test_conf) cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA cls.params[cf.PROCESSOR] = 1 # gamma file_list = cf.parse_namelist( os.path.join(common.SML_TEST_GAMMA, 'ifms_17')) cls.params[cf.IFG_FILE_LIST] = tempfile.mktemp(dir=cls.tif_dir_serial) # write a short filelist with only 3 gamma unws with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp: for f in file_list[:2]: fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n') cls.params[cf.OUT_DIR] = cls.tif_dir_serial cls.params[cf.PARALLEL] = 0 cls.params[cf.REF_EST_METHOD] = 2 cls.params[cf.IFG_LKSX] = 1 cls.params[cf.IFG_LKSY] = 1 cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA cls.params[cf.APS_INCIDENCE_MAP] = common.SML_TEST_INCIDENCE # base_unw_paths need to be geotiffed and multilooked by run_prepifg base_unw_paths = run_pyrate.original_ifg_paths( cls.params[cf.IFG_FILE_LIST]) # add dem base_unw_paths.append(common.SML_TEST_DEM_GAMMA) # add incidence base_unw_paths.append(common.SML_TEST_INCIDENCE) xlks, ylks, crop = run_pyrate.transform_params(cls.params) cls.params_mpi = copy.copy(cls.params) cls.params_mpi[cf.OUT_DIR] = cls.tif_dir_mpi # dest_paths are tifs that have been geotif converted and multilooked run_prepifg.gamma_prepifg(base_unw_paths, cls.params) run_prepifg.gamma_prepifg(base_unw_paths, cls.params_mpi) # removed incidence as we don't want it in ifgs list base_unw_paths.pop() # removed dem as we don't want it in ifgs list base_unw_paths.pop() dest_paths = run_pyrate.get_dest_paths(base_unw_paths, crop, cls.params, xlks) dest_paths_mpi = run_pyrate.get_dest_paths(base_unw_paths, crop, cls.params_mpi, xlks) run_pyrate.process_ifgs(dest_paths, cls.params) cls.ifgs_serial = common.small_data_setup(datafiles=dest_paths) cls.conf_mpi = tempfile.mktemp('.conf', dir=cls.tif_dir_mpi) cf.write_config_file(cls.params_mpi, cls.conf_mpi) str = 'mpirun -np 4 python pyrate/nci/run_pyrate_pypar.py ' + \ cls.conf_mpi cmd = str.split() subprocess.check_call(cmd) str = 'mpirun -np 4 python pyrate/nci/run_pyrate_pypar_2.py ' + \ cls.conf_mpi cmd = str.split() subprocess.check_call(cmd) str = 'mpirun -np 4 python pyrate/nci/run_pyrate_pypar_3.py ' + \ cls.conf_mpi cmd = str.split() subprocess.check_call(cmd) cls.ifgs_mpi = common.small_data_setup(datafiles=dest_paths_mpi)