def test_new_config_file_and_original_match(self): params = config.get_config_params(TEST_CONF_GAMMA) temp_config = tempfile.mktemp(suffix='.conf') config.write_config_file(params, temp_config) new_params = config.get_config_params(temp_config) self.maxDiff = None self.assertDictEqual(params, new_params) os.remove(temp_config)
def setUpClass(cls): # testing constants2 cls.BASE_DIR = tempfile.mkdtemp() cls.BASE_OUT_DIR = join(cls.BASE_DIR, 'out') cls.BASE_DEM_DIR = join(cls.BASE_DIR, 'dem') cls.BASE_DEM_FILE = join(cls.BASE_DEM_DIR, 'roipac_test_trimmed.tif') try: # copy source data (treat as prepifg already run) os.makedirs(cls.BASE_OUT_DIR) for path in glob.glob(join(common.SML_TEST_TIF, '*')): dest = join(cls.BASE_OUT_DIR, os.path.basename(path)) shutil.copy(path, dest) os.chmod(dest, 0o660) os.makedirs(cls.BASE_DEM_DIR) orig_dem = common.SML_TEST_DEM_TIF os.symlink(orig_dem, cls.BASE_DEM_FILE) os.chdir(cls.BASE_DIR) params = config.get_config_params(common.TEST_CONF_ROIPAC) params[cf.OUT_DIR] = cls.BASE_OUT_DIR params[cf.PROCESSOR] = 0 # roipac params[cf.APS_CORRECTION] = 0 paths = glob.glob(join(cls.BASE_OUT_DIR, 'geo_*-*.tif')) params[cf.PARALLEL] = False run_pyrate.process_ifgs(sorted(paths), params, 2, 2) if not hasattr(cls, 'ifgs'): cls.ifgs = get_ifgs(out_dir=cls.BASE_OUT_DIR) except: # revert working dir & avoid paths busting other tests os.chdir(CURRENT_DIR) raise
def common_check(self, ele, inc): os.path.exists(self.conf_file) params = cf.get_config_params(self.conf_file) sys.argv = ['dummy', self.conf_file] run_prepifg.main(params) # test 17 geotiffs created geotifs = glob.glob(os.path.join(self.base_dir, '*_unw.tif')) self.assertEqual(17, len(geotifs)) # test dem geotiff created demtif = glob.glob(os.path.join(self.base_dir, '*_dem.tif')) self.assertEqual(1, len(demtif)) # elevation/incidence file ele = glob.glob( os.path.join(self.base_dir, '*utm_{ele}.tif'.format(ele=ele)))[0] self.assertTrue(os.path.exists(ele)) # mlooked tifs mlooked_tifs = [ f for f in glob.glob(os.path.join(self.base_dir, '*.tif')) if "cr" in f and "rlks" in f ] # 19 including 17 ifgs, 1 dem and one incidence self.assertEqual(19, len(mlooked_tifs)) inc = glob.glob( os.path.join(self.base_dir, '*utm_{inc}.tif'.format(inc=inc))) self.assertEqual(0, len(inc))
def test_read_param_file_missing_value(): # ensure the parser can handle blank option values conf_path = join(SML_TEST_CONF, 'pyrate2.conf') params = config.get_config_params(conf_path) assert params[config.REFX] == -1 assert params[config.REFY] == -1
def setUp(self): self.BASE_DIR = tempfile.mkdtemp() self.params = cf.get_config_params(TEST_CONF_ROIPAC) # change to orbital error correction method 2 self.params[cf.ORBITAL_FIT_METHOD] = NETWORK_METHOD self.params[cf.ORBITAL_FIT_LOOKS_X] = 1 self.params[cf.ORBITAL_FIT_LOOKS_Y] = 1 data_paths = [ os.path.join(SML_TEST_TIF, p) for p in small_ifg_file_list() ] self.new_data_paths = [ os.path.join(self.BASE_DIR, os.path.basename(d)) for d in data_paths ] for d in data_paths: d_copy = os.path.join(self.BASE_DIR, os.path.basename(d)) shutil.copy(d, d_copy) os.chmod(d_copy, 0o660) self.ifgs = small_data_setup(datafiles=self.new_data_paths) for i in self.ifgs: if not i.is_open: i.open() if not i.nan_converted: i.convert_to_nans() if not i.mm_converted: i.convert_to_mm() i.write_modified_phase()
def test_read_param_file_missing_option(): # ensure the parser can handle missing option fields conf_path = join(SML_TEST_CONF, 'pyrate1.conf') params = config.get_config_params(conf_path) assert params[config.REFX] == -1 assert params[config.REFY] == -1
def setUpClass(cls): cls.tif_dir = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_GAMMA # change the required params cls.params = cf.get_config_params(cls.test_conf) cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA cls.params[cf.PROCESSOR] = 1 # gamma file_list = list(cf.parse_namelist(os.path.join(common.SML_TEST_GAMMA, 'ifms_17'))) fd, cls.params[cf.IFG_FILE_LIST] = tempfile.mkstemp(suffix='.conf', dir=cls.tif_dir) os.close(fd) # write a short filelist with only 3 gamma unws with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp: for f in file_list[:3]: fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n') cls.params[cf.OUT_DIR] = cls.tif_dir cls.params[cf.PARALLEL] = 0 cls.params[cf.REF_EST_METHOD] = 1 cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA # base_unw_paths need to be geotiffed and multilooked by run_prepifg cls.base_unw_paths = cf.original_ifg_paths( cls.params[cf.IFG_FILE_LIST]) cls.base_unw_paths.append(common.SML_TEST_DEM_GAMMA) xlks, ylks, crop = cf.transform_params(cls.params) # dest_paths are tifs that have been geotif converted and multilooked run_prepifg.gamma_prepifg(cls.base_unw_paths, cls.params) cls.base_unw_paths.pop() # removed dem as we don't want it in ifgs dest_paths = cf.get_dest_paths( cls.base_unw_paths, crop, cls.params, xlks) cls.ifgs = common.small_data_setup(datafiles=dest_paths)
def setUp(self): self.ifgs = small_data_setup() self.params = cf.get_config_params(TEST_CONF_ROIPAC) self.params[cf.REFNX] = REFNX self.params[cf.REFNY] = REFNY self.params[cf.REF_CHIP_SIZE] = CHIPSIZE self.params[cf.REF_MIN_FRAC] = MIN_FRAC self.params[cf.PARALLEL] = PARALLEL
def setUpClass(cls): params = cf.get_config_params(common.TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['run_prepifg.py', common.TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir run_prepifg.main(params) params[cf.REF_EST_METHOD] = 2 xlks, ylks, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) # start run_pyrate copy ifgs = common.pre_prepare_ifgs(dest_paths, params) mst_grid = common.mst_calculation(dest_paths, params) refx, refy = run_pyrate._ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.orbital.remove_orbital_error(ifgs, params) ifgs = common.prepare_ifgs_without_phase(dest_paths, params) _, ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy) r_dist = covariance.RDist(ifgs[0])() maxvar = [covariance.cvd(i, params, r_dist)[0] for i in ifgs] vcmt = covariance.get_vcmt(ifgs, maxvar) params[cf.TIME_SERIES_METHOD] = 1 params[cf.PARALLEL] = 0 # Calculate time series cls.tsincr_0, cls.tscum_0, _ = common.calculate_time_series( ifgs, params, vcmt, mst=mst_grid) params[cf.PARALLEL] = 1 cls.tsincr_1, cls.tscum_1, cls.tsvel_1 = \ common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid) params[cf.PARALLEL] = 2 cls.tsincr_2, cls.tscum_2, cls.tsvel_2 = \ common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid) # load the matlab data ts_dir = os.path.join(common.SML_TEST_DIR, 'matlab_time_series') tsincr_path = os.path.join(ts_dir, 'ts_incr_interp0_method1.csv') ts_incr = np.genfromtxt(tsincr_path) # the matlab tsvel return is a bit pointless and not tested here # tserror is not returned # tserr_path = os.path.join(SML_TIME_SERIES_DIR, # 'ts_error_interp0_method1.csv') # ts_err = np.genfromtxt(tserr_path, delimiter=',') tscum_path = os.path.join(ts_dir, 'ts_cum_interp0_method1.csv') ts_cum = np.genfromtxt(tscum_path) cls.ts_incr = np.reshape(ts_incr, newshape=cls.tsincr_0.shape, order='F') cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F')
def prepifg(config_file): """ Convert input files to geotiff and perform multilooking (resampling) and/or cropping """ config_file = os.path.abspath(config_file) params = cf.get_config_params(config_file) log.info('This job was run with the following parameters:') log.info(json.dumps(params, indent=4, sort_keys=True)) run_prepifg.main(params)
def setUpClass(cls): cls.tif_dir = tempfile.mkdtemp() cls.tif_dir_method2 = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_ROIPAC # change the required params cls.params = cf.get_config_params(cls.test_conf) cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA cls.params[cf.PROCESSOR] = 1 # gamma file_list = cf.parse_namelist( os.path.join(common.SML_TEST_GAMMA, 'ifms_17')) cls.params[cf.IFG_FILE_LIST] = tempfile.mktemp(dir=cls.tif_dir) # write a short filelist with only 3 gamma unws with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp: for f in file_list[:2]: fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n') cls.params[cf.OUT_DIR] = cls.tif_dir cls.params[cf.PARALLEL] = True cls.params[cf.REF_EST_METHOD] = 1 cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA cls.params[cf.APS_INCIDENCE_MAP] = common.SML_TEST_INCIDENCE # base_unw_paths need to be geotiffed and multilooked by run_prepifg base_unw_paths = run_pyrate.original_ifg_paths( cls.params[cf.IFG_FILE_LIST]) # add dem base_unw_paths.append(common.SML_TEST_DEM_GAMMA) # add incidence base_unw_paths.append(common.SML_TEST_INCIDENCE) xlks, ylks, crop = run_pyrate.transform_params(cls.params) import copy cls.params_method2 = copy.copy(cls.params) cls.params_method2[cf.OUT_DIR] = cls.tif_dir_method2 cls.params_method2[cf.APS_METHOD] = 2 # dest_paths are tifs that have been geotif converted and multilooked run_prepifg.gamma_prepifg(base_unw_paths, cls.params) run_prepifg.gamma_prepifg(base_unw_paths, cls.params_method2) # removed incidence as we don't want it in ifgs list base_unw_paths.pop() # removed dem as we don't want it in ifgs list base_unw_paths.pop() dest_paths = run_pyrate.get_dest_paths(base_unw_paths, crop, cls.params, xlks) cls.ifgs = common.small_data_setup(datafiles=dest_paths) dest_paths_m2 = run_pyrate.get_dest_paths(base_unw_paths, crop, cls.params_method2, xlks) cls.ifgs_method2 = common.small_data_setup(datafiles=dest_paths_m2) aps.remove_aps_delay(cls.ifgs, cls.params) aps.remove_aps_delay(cls.ifgs_method2, cls.params_method2)
def prepifg(config_file): """ Convert input files to geotiff and perform multilooking (resampling) and/or cropping. """ config_file = abspath(config_file) params = cf.get_config_params(config_file) if params[cf.LUIGI]: run_prepifg.main() else: run_prepifg.main(params)
def setUpClass(cls): rate_types = ['linrate', 'linerror', 'linsamples'] cls.tif_dir = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_GAMMA # change the required params params = cf.get_config_params(cls.test_conf) params[cf.OBS_DIR] = common.SML_TEST_GAMMA params[cf.PROCESSOR] = 1 # gamma params[cf.IFG_FILE_LIST] = os.path.join(common.SML_TEST_GAMMA, 'ifms_17') params[cf.OUT_DIR] = cls.tif_dir params[cf.PARALLEL] = 0 params[cf.APS_CORRECTION] = False params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) xlks, ylks, crop = cf.transform_params(params) # base_unw_paths need to be geotiffed and multilooked by run_prepifg base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) # dest_paths are tifs that have been geotif converted and multilooked cls.dest_paths = cf.get_dest_paths(base_unw_paths, crop, params, xlks) run_prepifg.gamma_prepifg(base_unw_paths, params) tiles = run_pyrate.get_tiles(cls.dest_paths[0], 3, 3) ifgs = common.small_data_setup() cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = \ run_pyrate.process_ifgs(cls.dest_paths, params, 3, 3) cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) cls.rate_p, cls.error_p, cls.samples_p = [ common.reconstruct_linrate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types ] # now create the non parallel version cls.tif_dir_s = tempfile.mkdtemp() params[cf.PARALLEL] = 0 params[cf.OUT_DIR] = cls.tif_dir_s params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) cls.dest_paths_s = cf.get_dest_paths(base_unw_paths, crop, params, xlks) run_prepifg.gamma_prepifg(base_unw_paths, params) cls.refpixel, cls.maxvar, cls.vcmt = \ run_pyrate.process_ifgs(cls.dest_paths_s, params, 3, 3) cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) cls.rate, cls.error, cls.samples = [ common.reconstruct_linrate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types ]
def test_spatio_temporal_filter(): import tempfile from pyrate import shared from os.path import basename, join from collections import OrderedDict from pyrate.scripts import run_prepifg from osgeo import gdal from pyrate import ifgconstants as ifc ifg_out = sio.loadmat( os.path.join(SML_TEST_DIR, 'matlab_aps', 'ifg_spatio_temp_out.mat'))['ifg'] tsincr = sio.loadmat( os.path.join(SML_TEST_DIR, 'matlab_aps', 'tsincr_svd.mat'))['tsincr'] params = cf.get_config_params(os.path.join(TEST_CONF_GAMMA)) params[cf.OUT_DIR] = tempfile.mkdtemp() params[cf.TMPDIR] = join(params[cf.OUT_DIR], cf.TMPDIR) shared.mkdir_p(params[cf.TMPDIR]) params[cf.SLPF_METHOD] = 2 params[cf.SLPF_CUTOFF] = 0 params[cf.SLPF_ORDER] = 1 params[cf.SLPF_NANFILL] = 0 params[cf.TLPF_METHOD] = 3 params[cf.TLPF_CUTOFF] = 0.25 params[cf.TLPF_PTHR] = 5 ifgs = small_data_setup() _ = [ifgs_pk.pop(k) for k in ['gt', 'epochlist', 'md', 'wkt']] preread_ifgs = { join(params[cf.OUT_DIR], basename(k)): v for k, v in ifgs_pk.items() } preread_ifgs = OrderedDict(sorted(preread_ifgs.items())) run_prepifg.main(params) for k, v in preread_ifgs.items(): v.path = join(params[cf.OUT_DIR], basename(k)) preread_ifgs[k] = v ifg = ifgs[0] ifg.x_size = xpsize ifg.y_size = ypsize spatio_temporal_filter(tsincr, ifg, params, preread_ifgs) for i in ifgs: i.close() for (p, v), i in zip(preread_ifgs.items(), range(ifg_out.shape[2])): ds = gdal.Open(p) metadata = ds.GetMetadata() assert ifc.PYRATE_APS_ERROR in metadata assert metadata[ifc.PYRATE_APS_ERROR] == ifc.APS_REMOVED arr = ds.GetRasterBand(1).ReadAsArray() np.testing.assert_array_almost_equal(arr, ifg_out[:, :, i], decimal=3)
def test_inc_vs_ele_maps_ele_provided(self): self.make_input_files(ele=common.SML_TEST_ELEVATION) assert os.path.exists(self.conf_file) params = config.get_config_params(self.conf_file) # incidence variables self.assertIn(config.APS_INCIDENCE_MAP, params.keys()) self.assertIn(config.APS_INCIDENCE_EXT, params.keys()) self.assertIsNone(params[config.APS_INCIDENCE_MAP]) self.assertIsNone(params[config.APS_INCIDENCE_EXT]) # elevation variables self.assertIn(config.APS_ELEVATION_MAP, params.keys()) self.assertIsNotNone(params[config.APS_ELEVATION_MAP]) self.assertIn(config.APS_ELEVATION_EXT, params.keys()) self.assertIn(config.APS_ELEVATION_MAP, params.keys())
def setUp(self): self.ifg_paths = small_ifg_file_list() self.params = cf.get_config_params(TEST_CONF_ROIPAC) self.params[cf.PARALLEL] = False self.params[cf.OUT_DIR] = tempfile.mkdtemp() self.params_alt_ref_frac = copy.copy(self.params) self.params_alt_ref_frac[cf.REF_MIN_FRAC] = 0.5 self.params_all_2s = copy.copy(self.params) self.params_all_2s[cf.REFNX] = 2 self.params_all_2s[cf.REFNY] = 2 self.params_chipsize_15 = copy.copy(self.params_all_2s) self.params_chipsize_15[cf.REF_CHIP_SIZE] = 15 self.params_all_1s = copy.copy(self.params) self.params_all_1s[cf.REFNX] = 1 self.params_all_1s[cf.REFNY] = 1 self.params_all_1s[cf.REF_MIN_FRAC] = 0.7
def setUp(self): self.BASE_DIR = tempfile.mkdtemp() self.params = cf.get_config_params(TEST_CONF_ROIPAC) # change to orbital error correction method 1 self.params[cf.ORBITAL_FIT_METHOD] = INDEPENDENT_METHOD self.params[cf.ORBITAL_FIT_LOOKS_X] = 2 self.params[cf.ORBITAL_FIT_LOOKS_Y] = 2 self.params[cf.PARALLEL] = False data_paths = [os.path.join(SML_TEST_TIF, p) for p in IFMS16] self.ifg_paths = [ os.path.join(self.BASE_DIR, os.path.basename(d)) for d in data_paths ] for d in data_paths: shutil.copy(d, os.path.join(self.BASE_DIR, os.path.basename(d)))
def setUpClass(cls): params = cf.get_config_params(TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['run_prepifg.py', TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir params[cf.REF_EST_METHOD] = 2 run_prepifg.main(params) xlks, ylks, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) ifgs = shared.pre_prepare_ifgs(dest_paths, params) refx, refy = run_pyrate.ref_pixel_calc(dest_paths, params) pyrate.orbital.remove_orbital_error(ifgs, params) ifgs = prepare_ifgs_without_phase(dest_paths, params) _, ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy) # Calculate interferogram noise cls.maxvar = [cvd(i, params)[0] for i in ifgs] cls.vcmt = get_vcmt(ifgs, cls.maxvar)
def setUpClass(cls): params = cf.get_config_params(common.TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['run_prepifg.py', common.TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir run_prepifg.main(params) params[cf.OUT_DIR] = cls.temp_out_dir params[cf.REF_EST_METHOD] = 2 params[cf.PARALLEL] = False xlks, ylks, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) # start run_pyrate copy ifgs = shared.pre_prepare_ifgs(dest_paths, params) mst_grid = tests.common.mst_calculation(dest_paths, params) # Estimate reference pixel location refx, refy = run_pyrate.ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.orbital.remove_orbital_error(ifgs, params) for i in ifgs: i.close() ifgs = shared.pre_prepare_ifgs(dest_paths, params) cls.ref_phs, cls.ifgs = estimate_ref_phase(ifgs, params, refx, refy) # end run_pyrate copy for i in ifgs: i.close()
def setUpClass(cls): params = config.get_config_params(common.TEST_CONF_ROIPAC) cls.tmp_dir = tempfile.mkdtemp() shared.copytree(common.SML_TEST_TIF, cls.tmp_dir) tifs = glob.glob(os.path.join(cls.tmp_dir, "*.tif")) for t in tifs: os.chmod(t, 0o644) small_ifgs = common.small_data_setup(datafiles=tifs) ifg_paths = [i.data_path for i in small_ifgs] cls.ifg_ret = shared.pre_prepare_ifgs(ifg_paths, params=params) for i in cls.ifg_ret: i.close() nan_conversion = params[cf.NAN_CONVERSION] # prepare a second set cls.tmp_dir2 = tempfile.mkdtemp() shared.copytree(common.SML_TEST_TIF, cls.tmp_dir2) tifs = glob.glob(os.path.join(cls.tmp_dir2, "*.tif")) for t in tifs: os.chmod(t, 0o644) small_ifgs = common.small_data_setup(datafiles=tifs) ifg_paths = [i.data_path for i in small_ifgs] cls.ifgs = [shared.Ifg(p) for p in ifg_paths] for i in cls.ifgs: if not i.is_open: i.open(readonly=False) if nan_conversion: # nan conversion happens here in networkx mst i.nodata_value = params[cf.NO_DATA_VALUE] i.convert_to_nans() if not i.mm_converted: i.convert_to_mm() i.close()
def setUpClass(cls): params = cf.get_config_params(TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['run_prepifg.py', TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir params[cf.TMPDIR] = os.path.join(cls.temp_out_dir, cf.TMPDIR) shared.mkdir_p(params[cf.TMPDIR]) params[cf.REF_EST_METHOD] = 2 run_prepifg.main(params) cls.params = params xlks, ylks, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) ifgs = common.pre_prepare_ifgs(dest_paths, params) refx, refy = run_pyrate._ref_pixel_calc(dest_paths, params) pyrate.orbital.remove_orbital_error(ifgs, params) ifgs = prepare_ifgs_without_phase(dest_paths, params) _, cls.ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy) r_dist = RDist(ifgs[0])() # Calculate interferogram noise cls.maxvar = [cvd(i, params, r_dist, calc_alpha=True, save_acg=True, write_vals=True)[0] for i in ifgs] cls.vcmt = get_vcmt(ifgs, cls.maxvar)
# tsincr matrix from matlab using svd timeseries method tsincr_svd = sio.loadmat( os.path.join(SML_TEST_DIR, 'matlab_aps', 'tsincr_svd.mat')) # prepread ifgs pickle file preread_pk = 'preread_ifgs.pk' if PY3 else 'preread_ifgs_py2.pk' ifgs_pk = os.path.join(SML_TEST_DIR, 'matlab_aps', preread_pk) ifgs_pk = pickle.load(open(ifgs_pk, 'rb')) # tlp filter output from matlab ts_hp = sio.loadmat(os.path.join(SML_TEST_DIR, 'matlab_aps', 'ts_hp.mat')) ts_hp_m2 = sio.loadmat(os.path.join(SML_TEST_DIR, 'matlab_aps', 'ts_hp_m2.mat')) ts_hp_m3 = sio.loadmat(os.path.join(SML_TEST_DIR, 'matlab_aps', 'ts_hp_m3.mat')) epochlist = get_epochs(ifgs_pk)[0] params = cf.get_config_params(os.path.join(TEST_CONF_GAMMA)) @pytest.fixture(params=[1, 2, 3]) def tlpfilter_method(request): return request.param @pytest.fixture(params=[1, 2]) def slpfilter_method(request): return request.param def test_tlpfilter_matlab(tlpfilter_method): params[cf.TLPF_METHOD] = tlpfilter_method ts = ts_hp if tlpfilter_method == 1 \
def params(conf_file): return cf.get_config_params(conf_file)
def setUp(self): self.ifgs = small_data_setup() self.params = cf.get_config_params(common.TEST_CONF_ROIPAC)
def setUpClass(cls): cls.tif_dir_inc = tempfile.mkdtemp() cls.tif_dir_ele = tempfile.mkdtemp() cls.tif_dir_ele_par = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_ROIPAC # change the required params cls.params_inc = cf.get_config_params(cls.test_conf) cls.params_inc[cf.OBS_DIR] = common.SML_TEST_GAMMA cls.params_inc[cf.PROCESSOR] = 1 # gamma file_list = cf.parse_namelist( os.path.join(common.SML_TEST_GAMMA, 'ifms_17')) # config file cls.params_inc[cf.IFG_FILE_LIST] = tempfile.mktemp(dir=cls.tif_dir_inc) # write a short filelist with only 3 gamma unws with open(cls.params_inc[cf.IFG_FILE_LIST], 'w') as fp: for f in file_list[:2]: fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n') cls.params_inc[cf.OUT_DIR] = cls.tif_dir_inc cls.params_inc[cf.PARALLEL] = 0 cls.params_inc[cf.REF_EST_METHOD] = 1 cls.params_inc[cf.APS_METHOD] = 2 cls.params_inc[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA cls.params_inc[cf.APS_INCIDENCE_MAP] = common.SML_TEST_INCIDENCE run_prepifg.main(cls.params_inc) # now create the config for the elevation_map case cls.params_ele = copy.copy(cls.params_inc) cls.params_ele[cf.OUT_DIR] = cls.tif_dir_ele cls.params_ele[cf.APS_METHOD] = 2 cls.params_ele[cf.APS_INCIDENCE_MAP] = None cls.params_ele[cf.APS_INCIDENCE_EXT] = None cls.params_ele[cf.APS_ELEVATION_MAP] = common.SML_TEST_ELEVATION cls.params_ele[cf.APS_ELEVATION_EXT] = 'lv_theta' run_prepifg.main(cls.params_ele) ptn = re.compile(r'\d{8}') dest_paths_inc = [ f for f in glob.glob(os.path.join(cls.tif_dir_inc, '*.tif')) if ("cr" in f) and ("rlks" in f) and ( len(re.findall(ptn, os.path.basename(f))) == 2) ] cls.ifgs_inc = common.small_data_setup(datafiles=dest_paths_inc) dest_paths_ele = [ f for f in glob.glob(os.path.join(cls.tif_dir_ele, '*.tif')) if "cr" in f and "rlks" in f and ( len(re.findall(ptn, os.path.basename(f))) == 2) ] cls.ifgs_ele = common.small_data_setup(datafiles=dest_paths_ele) # now create the config for the elevation map parallel case cls.params_ele_par = copy.copy(cls.params_ele) cls.params_ele_par[cf.OUT_DIR] = cls.tif_dir_ele_par cls.params_ele_par[cf.PARALLEL] = True run_prepifg.main(cls.params_ele_par) dest_paths_ele_par = \ [f for f in glob.glob(os.path.join(cls.tif_dir_ele_par, '*.tif')) if "cr" in f and "rlks" in f and (len(re.findall(ptn, os.path.basename(f))) == 2)] cls.ifgs_ele_par = common.small_data_setup( datafiles=dest_paths_ele_par) aps.remove_aps_delay(cls.ifgs_inc, cls.params_inc) aps.remove_aps_delay(cls.ifgs_ele, cls.params_ele) aps.remove_aps_delay(cls.ifgs_ele_par, cls.params_ele_par)
def setUpClass(cls): cls.tif_dir_serial = tempfile.mkdtemp() cls.tif_dir_mpi = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_ROIPAC # change the required params cls.params = cf.get_config_params(cls.test_conf) cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA cls.params[cf.PROCESSOR] = 1 # gamma file_list = cf.parse_namelist( os.path.join(common.SML_TEST_GAMMA, 'ifms_17')) cls.params[cf.IFG_FILE_LIST] = tempfile.mktemp(dir=cls.tif_dir_serial) # write a short filelist with only 3 gamma unws with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp: for f in file_list[:2]: fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n') cls.params[cf.OUT_DIR] = cls.tif_dir_serial cls.params[cf.PARALLEL] = 0 cls.params[cf.REF_EST_METHOD] = 2 cls.params[cf.IFG_LKSX] = 1 cls.params[cf.IFG_LKSY] = 1 cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA cls.params[cf.APS_INCIDENCE_MAP] = common.SML_TEST_INCIDENCE # base_unw_paths need to be geotiffed and multilooked by run_prepifg base_unw_paths = run_pyrate.original_ifg_paths( cls.params[cf.IFG_FILE_LIST]) # add dem base_unw_paths.append(common.SML_TEST_DEM_GAMMA) # add incidence base_unw_paths.append(common.SML_TEST_INCIDENCE) xlks, ylks, crop = run_pyrate.transform_params(cls.params) cls.params_mpi = copy.copy(cls.params) cls.params_mpi[cf.OUT_DIR] = cls.tif_dir_mpi # dest_paths are tifs that have been geotif converted and multilooked run_prepifg.gamma_prepifg(base_unw_paths, cls.params) run_prepifg.gamma_prepifg(base_unw_paths, cls.params_mpi) # removed incidence as we don't want it in ifgs list base_unw_paths.pop() # removed dem as we don't want it in ifgs list base_unw_paths.pop() dest_paths = run_pyrate.get_dest_paths(base_unw_paths, crop, cls.params, xlks) dest_paths_mpi = run_pyrate.get_dest_paths(base_unw_paths, crop, cls.params_mpi, xlks) run_pyrate.process_ifgs(dest_paths, cls.params) cls.ifgs_serial = common.small_data_setup(datafiles=dest_paths) cls.conf_mpi = tempfile.mktemp('.conf', dir=cls.tif_dir_mpi) cf.write_config_file(cls.params_mpi, cls.conf_mpi) str = 'mpirun -np 4 python pyrate/nci/run_pyrate_pypar.py ' + \ cls.conf_mpi cmd = str.split() subprocess.check_call(cmd) str = 'mpirun -np 4 python pyrate/nci/run_pyrate_pypar_2.py ' + \ cls.conf_mpi cmd = str.split() subprocess.check_call(cmd) str = 'mpirun -np 4 python pyrate/nci/run_pyrate_pypar_3.py ' + \ cls.conf_mpi cmd = str.split() subprocess.check_call(cmd) cls.ifgs_mpi = common.small_data_setup(datafiles=dest_paths_mpi)
def test_read_param_file(): params = config.get_config_params(TEST_CONF_ROIPAC) for k in params.keys(): assert k and len(k) > 1 assert params[k] != '' assert not k.endswith(":") # are the colons removed?
def test_write_config_file(self): params = config.get_config_params(TEST_CONF_GAMMA) temp_config = tempfile.mktemp(suffix='.conf') config.write_config_file(params, temp_config) self.assertTrue(os.path.exists(temp_config)) os.remove(temp_config)
def setUpClass(cls): params = cf.get_config_params(TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['run_prepifg.py', TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) shared.mkdir_p(params[cf.TMPDIR]) run_prepifg.main(params) params[cf.REF_EST_METHOD] = 2 xlks, _, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) # start run_pyrate copy ifgs = pre_prepare_ifgs(dest_paths, params) mst_grid = tests.common.mst_calculation(dest_paths, params) refx, refy = run_pyrate._ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.orbital.remove_orbital_error(ifgs, params) ifgs = prepare_ifgs_without_phase(dest_paths, params) _, ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy) r_dist = vcm_module.RDist(ifgs[0])() maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in ifgs] vcmt = vcm_module.get_vcmt(ifgs, maxvar) # Calculate linear rate map params[cf.PARALLEL] = 1 cls.rate, cls.error, cls.samples = tests.common.calculate_linear_rate( ifgs, params, vcmt, mst_mat=mst_grid) params[cf.PARALLEL] = 2 cls.rate_2, cls.error_2, cls.samples_2 = \ tests.common.calculate_linear_rate(ifgs, params, vcmt, mst_mat=mst_grid) params[cf.PARALLEL] = 0 # Calculate linear rate map cls.rate_s, cls.error_s, cls.samples_s = \ tests.common.calculate_linear_rate(ifgs, params, vcmt, mst_mat=mst_grid) matlab_linrate_dir = os.path.join(SML_TEST_DIR, 'matlab_linrate') cls.rate_matlab = np.genfromtxt(os.path.join(matlab_linrate_dir, 'stackmap.csv'), delimiter=',') cls.error_matlab = np.genfromtxt(os.path.join(matlab_linrate_dir, 'errormap.csv'), delimiter=',') cls.samples_matlab = np.genfromtxt(os.path.join( matlab_linrate_dir, 'coh_sta.csv'), delimiter=',')
def setUp(self): self.conf_path = TEST_CONF_ROIPAC self.params = config.get_config_params(self.conf_path)