def common_check(self, ele, inc): os.path.exists(self.conf_file) params = cf.get_config_params(self.conf_file) sys.argv = ['dummy', self.conf_file] run_prepifg.main(params) # test 17 geotiffs created geotifs = glob.glob(os.path.join(self.base_dir, '*_unw.tif')) self.assertEqual(17, len(geotifs)) # test dem geotiff created demtif = glob.glob(os.path.join(self.base_dir, '*_dem.tif')) self.assertEqual(1, len(demtif)) # elevation/incidence file ele = glob.glob( os.path.join(self.base_dir, '*utm_{ele}.tif'.format(ele=ele)))[0] self.assertTrue(os.path.exists(ele)) # mlooked tifs mlooked_tifs = [ f for f in glob.glob(os.path.join(self.base_dir, '*.tif')) if "cr" in f and "rlks" in f ] # 19 including 17 ifgs, 1 dem and one incidence self.assertEqual(19, len(mlooked_tifs)) inc = glob.glob( os.path.join(self.base_dir, '*utm_{inc}.tif'.format(inc=inc))) self.assertEqual(0, len(inc))
def check_roipac(self): self.make_roipac_input_files(self.dataPaths, 'WGS84') sys.argv = ['pyrate', 'prepifg', self.confFile] run_prepifg.main() for path in self.expPaths: self.assertTrue(os.path.exists(path), '{} does not exist'.format(path))
def setUpClass(cls): params = cf.get_config_params(common.TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['run_prepifg.py', common.TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir run_prepifg.main(params) params[cf.REF_EST_METHOD] = 2 xlks, ylks, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) # start run_pyrate copy ifgs = common.pre_prepare_ifgs(dest_paths, params) mst_grid = common.mst_calculation(dest_paths, params) refx, refy = run_pyrate._ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.orbital.remove_orbital_error(ifgs, params) ifgs = common.prepare_ifgs_without_phase(dest_paths, params) _, ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy) r_dist = covariance.RDist(ifgs[0])() maxvar = [covariance.cvd(i, params, r_dist)[0] for i in ifgs] vcmt = covariance.get_vcmt(ifgs, maxvar) params[cf.TIME_SERIES_METHOD] = 1 params[cf.PARALLEL] = 0 # Calculate time series cls.tsincr_0, cls.tscum_0, _ = common.calculate_time_series( ifgs, params, vcmt, mst=mst_grid) params[cf.PARALLEL] = 1 cls.tsincr_1, cls.tscum_1, cls.tsvel_1 = \ common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid) params[cf.PARALLEL] = 2 cls.tsincr_2, cls.tscum_2, cls.tsvel_2 = \ common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid) # load the matlab data ts_dir = os.path.join(common.SML_TEST_DIR, 'matlab_time_series') tsincr_path = os.path.join(ts_dir, 'ts_incr_interp0_method1.csv') ts_incr = np.genfromtxt(tsincr_path) # the matlab tsvel return is a bit pointless and not tested here # tserror is not returned # tserr_path = os.path.join(SML_TIME_SERIES_DIR, # 'ts_error_interp0_method1.csv') # ts_err = np.genfromtxt(tserr_path, delimiter=',') tscum_path = os.path.join(ts_dir, 'ts_cum_interp0_method1.csv') ts_cum = np.genfromtxt(tscum_path) cls.ts_incr = np.reshape(ts_incr, newshape=cls.tsincr_0.shape, order='F') cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F')
def prepifg(config_file): """ Convert input files to geotiff and perform multilooking (resampling) and/or cropping """ config_file = os.path.abspath(config_file) params = cf.get_config_params(config_file) log.info('This job was run with the following parameters:') log.info(json.dumps(params, indent=4, sort_keys=True)) run_prepifg.main(params)
def prepifg(config_file): """ Convert input files to geotiff and perform multilooking (resampling) and/or cropping. """ config_file = abspath(config_file) params = cf.get_config_params(config_file) if params[cf.LUIGI]: run_prepifg.main() else: run_prepifg.main(params)
def test_spatio_temporal_filter(): import tempfile from pyrate import shared from os.path import basename, join from collections import OrderedDict from pyrate.scripts import run_prepifg from osgeo import gdal from pyrate import ifgconstants as ifc ifg_out = sio.loadmat( os.path.join(SML_TEST_DIR, 'matlab_aps', 'ifg_spatio_temp_out.mat'))['ifg'] tsincr = sio.loadmat( os.path.join(SML_TEST_DIR, 'matlab_aps', 'tsincr_svd.mat'))['tsincr'] params = cf.get_config_params(os.path.join(TEST_CONF_GAMMA)) params[cf.OUT_DIR] = tempfile.mkdtemp() params[cf.TMPDIR] = join(params[cf.OUT_DIR], cf.TMPDIR) shared.mkdir_p(params[cf.TMPDIR]) params[cf.SLPF_METHOD] = 2 params[cf.SLPF_CUTOFF] = 0 params[cf.SLPF_ORDER] = 1 params[cf.SLPF_NANFILL] = 0 params[cf.TLPF_METHOD] = 3 params[cf.TLPF_CUTOFF] = 0.25 params[cf.TLPF_PTHR] = 5 ifgs = small_data_setup() _ = [ifgs_pk.pop(k) for k in ['gt', 'epochlist', 'md', 'wkt']] preread_ifgs = { join(params[cf.OUT_DIR], basename(k)): v for k, v in ifgs_pk.items() } preread_ifgs = OrderedDict(sorted(preread_ifgs.items())) run_prepifg.main(params) for k, v in preread_ifgs.items(): v.path = join(params[cf.OUT_DIR], basename(k)) preread_ifgs[k] = v ifg = ifgs[0] ifg.x_size = xpsize ifg.y_size = ypsize spatio_temporal_filter(tsincr, ifg, params, preread_ifgs) for i in ifgs: i.close() for (p, v), i in zip(preread_ifgs.items(), range(ifg_out.shape[2])): ds = gdal.Open(p) metadata = ds.GetMetadata() assert ifc.PYRATE_APS_ERROR in metadata assert metadata[ifc.PYRATE_APS_ERROR] == ifc.APS_REMOVED arr = ds.GetRasterBand(1).ReadAsArray() np.testing.assert_array_almost_equal(arr, ifg_out[:, :, i], decimal=3)
def test_prepifg_mpi(mpisync, get_config, tempdir, roipac_or_gamma, get_lks, get_crop): from tests.common import TEST_CONF_ROIPAC, TEST_CONF_GAMMA from os.path import join, basename if roipac_or_gamma == 1: params = get_config(TEST_CONF_GAMMA) else: params = get_config(TEST_CONF_ROIPAC) outdir = mpiops.run_once(tempdir) params[cf.OUT_DIR] = outdir params[cf.PARALLEL] = False params[cf.IFG_LKSX], params[cf.IFG_LKSY] = get_lks, get_lks params[cf.IFG_CROP_OPT] = get_crop if roipac_or_gamma == 1: params[cf.IFG_FILE_LIST] = join(common.SML_TEST_GAMMA, 'ifms_17') params[cf.OBS_DIR] = common.SML_TEST_GAMMA params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA params[cf.DEM_HEADER_FILE] = common.SML_TEST_DEM_HDR_GAMMA run_prepifg.main(params) if mpiops.rank == 0: if roipac_or_gamma == 1: params_s = get_config(TEST_CONF_GAMMA) else: params_s = get_config(TEST_CONF_ROIPAC) params_s[cf.OUT_DIR] = tempdir() params_s[cf.PARALLEL] = True params_s[cf.IFG_LKSX], params_s[cf.IFG_LKSY] = get_lks, get_lks params_s[cf.IFG_CROP_OPT] = get_crop if roipac_or_gamma == 1: base_unw_paths = glob.glob(join(common.SML_TEST_GAMMA, "*_utm.unw")) run_prepifg.gamma_prepifg(base_unw_paths, params_s) else: base_unw_paths = glob.glob(join(common.SML_TEST_OBS, "*.unw")) run_prepifg.roipac_prepifg(base_unw_paths, params_s) mpi_tifs = glob.glob(join(outdir, "*.tif")) serial_tifs = glob.glob(join(params[cf.OUT_DIR], "*.tif")) mpi_tifs.sort() serial_tifs.sort() # 17 geotifs, and 17 mlooked tifs assert len(mpi_tifs) == len(serial_tifs) for m_f, s_f in zip(mpi_tifs, serial_tifs): assert basename(m_f) == basename(s_f) shutil.rmtree(outdir) shutil.rmtree(params_s[cf.OUT_DIR])
def common_check(self, conf_file): data_paths = glob.glob(os.path.join(SML_TEST_GAMMA, "*_utm.unw")) self.make_input_files(data_paths) base_ifg_paths, dest_paths, params = cf.get_ifg_paths(conf_file) dest_base_ifgs = [ os.path.join( params[cf.OUT_DIR], os.path.basename(q).split('.')[0] + '_' + os.path.basename(q).split('.')[1] + '.tif') for q in base_ifg_paths ] sys.argv = ['pyrate', 'prepifg', conf_file] run_prepifg.main() for p, q in zip(dest_base_ifgs, dest_paths): self.assertTrue(os.path.exists(p), '{} does not exist'.format(p)) self.assertTrue(os.path.exists(q), '{} does not exist'.format(q))
def setUpClass(cls): params = cf.get_config_params(TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['run_prepifg.py', TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir params[cf.REF_EST_METHOD] = 2 run_prepifg.main(params) xlks, ylks, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) ifgs = shared.pre_prepare_ifgs(dest_paths, params) refx, refy = run_pyrate.ref_pixel_calc(dest_paths, params) pyrate.orbital.remove_orbital_error(ifgs, params) ifgs = prepare_ifgs_without_phase(dest_paths, params) _, ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy) # Calculate interferogram noise cls.maxvar = [cvd(i, params)[0] for i in ifgs] cls.vcmt = get_vcmt(ifgs, cls.maxvar)
def setUpClass(cls): params = cf.get_config_params(common.TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['run_prepifg.py', common.TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir run_prepifg.main(params) params[cf.OUT_DIR] = cls.temp_out_dir params[cf.REF_EST_METHOD] = 2 params[cf.PARALLEL] = False xlks, ylks, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) # start run_pyrate copy ifgs = shared.pre_prepare_ifgs(dest_paths, params) mst_grid = tests.common.mst_calculation(dest_paths, params) # Estimate reference pixel location refx, refy = run_pyrate.ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.orbital.remove_orbital_error(ifgs, params) for i in ifgs: i.close() ifgs = shared.pre_prepare_ifgs(dest_paths, params) cls.ref_phs, cls.ifgs = estimate_ref_phase(ifgs, params, refx, refy) # end run_pyrate copy for i in ifgs: i.close()
def setUpClass(cls): params = cf.get_config_params(TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['run_prepifg.py', TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir params[cf.TMPDIR] = os.path.join(cls.temp_out_dir, cf.TMPDIR) shared.mkdir_p(params[cf.TMPDIR]) params[cf.REF_EST_METHOD] = 2 run_prepifg.main(params) cls.params = params xlks, ylks, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) ifgs = common.pre_prepare_ifgs(dest_paths, params) refx, refy = run_pyrate._ref_pixel_calc(dest_paths, params) pyrate.orbital.remove_orbital_error(ifgs, params) ifgs = prepare_ifgs_without_phase(dest_paths, params) _, cls.ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy) r_dist = RDist(ifgs[0])() # Calculate interferogram noise cls.maxvar = [cvd(i, params, r_dist, calc_alpha=True, save_acg=True, write_vals=True)[0] for i in ifgs] cls.vcmt = get_vcmt(ifgs, cls.maxvar)
def setUpClass(cls): params = cf.get_config_params(TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['run_prepifg.py', TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) shared.mkdir_p(params[cf.TMPDIR]) run_prepifg.main(params) params[cf.REF_EST_METHOD] = 2 xlks, _, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) # start run_pyrate copy ifgs = pre_prepare_ifgs(dest_paths, params) mst_grid = tests.common.mst_calculation(dest_paths, params) refx, refy = run_pyrate._ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.orbital.remove_orbital_error(ifgs, params) ifgs = prepare_ifgs_without_phase(dest_paths, params) _, ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy) r_dist = vcm_module.RDist(ifgs[0])() maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in ifgs] vcmt = vcm_module.get_vcmt(ifgs, maxvar) # Calculate linear rate map params[cf.PARALLEL] = 1 cls.rate, cls.error, cls.samples = tests.common.calculate_linear_rate( ifgs, params, vcmt, mst_mat=mst_grid) params[cf.PARALLEL] = 2 cls.rate_2, cls.error_2, cls.samples_2 = \ tests.common.calculate_linear_rate(ifgs, params, vcmt, mst_mat=mst_grid) params[cf.PARALLEL] = 0 # Calculate linear rate map cls.rate_s, cls.error_s, cls.samples_s = \ tests.common.calculate_linear_rate(ifgs, params, vcmt, mst_mat=mst_grid) matlab_linrate_dir = os.path.join(SML_TEST_DIR, 'matlab_linrate') cls.rate_matlab = np.genfromtxt(os.path.join(matlab_linrate_dir, 'stackmap.csv'), delimiter=',') cls.error_matlab = np.genfromtxt(os.path.join(matlab_linrate_dir, 'errormap.csv'), delimiter=',') cls.samples_matlab = np.genfromtxt(os.path.join( matlab_linrate_dir, 'coh_sta.csv'), delimiter=',')
def setUpClass(cls): cls.tif_dir_inc = tempfile.mkdtemp() cls.tif_dir_ele = tempfile.mkdtemp() cls.tif_dir_ele_par = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_ROIPAC # change the required params cls.params_inc = cf.get_config_params(cls.test_conf) cls.params_inc[cf.OBS_DIR] = common.SML_TEST_GAMMA cls.params_inc[cf.PROCESSOR] = 1 # gamma file_list = cf.parse_namelist( os.path.join(common.SML_TEST_GAMMA, 'ifms_17')) # config file cls.params_inc[cf.IFG_FILE_LIST] = tempfile.mktemp(dir=cls.tif_dir_inc) # write a short filelist with only 3 gamma unws with open(cls.params_inc[cf.IFG_FILE_LIST], 'w') as fp: for f in file_list[:2]: fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n') cls.params_inc[cf.OUT_DIR] = cls.tif_dir_inc cls.params_inc[cf.PARALLEL] = 0 cls.params_inc[cf.REF_EST_METHOD] = 1 cls.params_inc[cf.APS_METHOD] = 2 cls.params_inc[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA cls.params_inc[cf.APS_INCIDENCE_MAP] = common.SML_TEST_INCIDENCE run_prepifg.main(cls.params_inc) # now create the config for the elevation_map case cls.params_ele = copy.copy(cls.params_inc) cls.params_ele[cf.OUT_DIR] = cls.tif_dir_ele cls.params_ele[cf.APS_METHOD] = 2 cls.params_ele[cf.APS_INCIDENCE_MAP] = None cls.params_ele[cf.APS_INCIDENCE_EXT] = None cls.params_ele[cf.APS_ELEVATION_MAP] = common.SML_TEST_ELEVATION cls.params_ele[cf.APS_ELEVATION_EXT] = 'lv_theta' run_prepifg.main(cls.params_ele) ptn = re.compile(r'\d{8}') dest_paths_inc = [ f for f in glob.glob(os.path.join(cls.tif_dir_inc, '*.tif')) if ("cr" in f) and ("rlks" in f) and ( len(re.findall(ptn, os.path.basename(f))) == 2) ] cls.ifgs_inc = common.small_data_setup(datafiles=dest_paths_inc) dest_paths_ele = [ f for f in glob.glob(os.path.join(cls.tif_dir_ele, '*.tif')) if "cr" in f and "rlks" in f and ( len(re.findall(ptn, os.path.basename(f))) == 2) ] cls.ifgs_ele = common.small_data_setup(datafiles=dest_paths_ele) # now create the config for the elevation map parallel case cls.params_ele_par = copy.copy(cls.params_ele) cls.params_ele_par[cf.OUT_DIR] = cls.tif_dir_ele_par cls.params_ele_par[cf.PARALLEL] = True run_prepifg.main(cls.params_ele_par) dest_paths_ele_par = \ [f for f in glob.glob(os.path.join(cls.tif_dir_ele_par, '*.tif')) if "cr" in f and "rlks" in f and (len(re.findall(ptn, os.path.basename(f))) == 2)] cls.ifgs_ele_par = common.small_data_setup( datafiles=dest_paths_ele_par) aps.remove_aps_delay(cls.ifgs_inc, cls.params_inc) aps.remove_aps_delay(cls.ifgs_ele, cls.params_ele) aps.remove_aps_delay(cls.ifgs_ele_par, cls.params_ele_par)