def common_check(self, ele, inc):
os.path.exists(self.conf_file)
params = cf.get_config_params(self.conf_file)
sys.argv = ['dummy', self.conf_file]
run_prepifg.main(params)
# test 17 geotiffs created
geotifs = glob.glob(os.path.join(self.base_dir, '*_unw.tif'))
self.assertEqual(17, len(geotifs))
# test dem geotiff created
demtif = glob.glob(os.path.join(self.base_dir, '*_dem.tif'))
self.assertEqual(1, len(demtif))
# elevation/incidence file
ele = glob.glob(
os.path.join(self.base_dir, '*utm_{ele}.tif'.format(ele=ele)))[0]
self.assertTrue(os.path.exists(ele))
# mlooked tifs
mlooked_tifs = [
f for f in glob.glob(os.path.join(self.base_dir, '*.tif'))
if "cr" in f and "rlks" in f
]
# 19 including 17 ifgs, 1 dem and one incidence
self.assertEqual(19, len(mlooked_tifs))
inc = glob.glob(
os.path.join(self.base_dir, '*utm_{inc}.tif'.format(inc=inc)))
self.assertEqual(0, len(inc))
def check_roipac(self):
self.make_roipac_input_files(self.dataPaths, 'WGS84')
sys.argv = ['pyrate', 'prepifg', self.confFile]
run_prepifg.main()
for path in self.expPaths:
self.assertTrue(os.path.exists(path),
'{} does not exist'.format(path))
def setUpClass(cls):
params = cf.get_config_params(common.TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['run_prepifg.py', common.TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
run_prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
# start run_pyrate copy
ifgs = common.pre_prepare_ifgs(dest_paths, params)
mst_grid = common.mst_calculation(dest_paths, params)
refx, refy = run_pyrate._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.orbital.remove_orbital_error(ifgs, params)
ifgs = common.prepare_ifgs_without_phase(dest_paths, params)
_, ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy)
r_dist = covariance.RDist(ifgs[0])()
maxvar = [covariance.cvd(i, params, r_dist)[0] for i in ifgs]
vcmt = covariance.get_vcmt(ifgs, maxvar)
params[cf.TIME_SERIES_METHOD] = 1
params[cf.PARALLEL] = 0
# Calculate time series
cls.tsincr_0, cls.tscum_0, _ = common.calculate_time_series(
ifgs, params, vcmt, mst=mst_grid)
params[cf.PARALLEL] = 1
cls.tsincr_1, cls.tscum_1, cls.tsvel_1 = \
common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid)
params[cf.PARALLEL] = 2
cls.tsincr_2, cls.tscum_2, cls.tsvel_2 = \
common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid)
# load the matlab data
ts_dir = os.path.join(common.SML_TEST_DIR, 'matlab_time_series')
tsincr_path = os.path.join(ts_dir, 'ts_incr_interp0_method1.csv')
ts_incr = np.genfromtxt(tsincr_path)
# the matlab tsvel return is a bit pointless and not tested here
# tserror is not returned
# tserr_path = os.path.join(SML_TIME_SERIES_DIR,
# 'ts_error_interp0_method1.csv')
# ts_err = np.genfromtxt(tserr_path, delimiter=',')
tscum_path = os.path.join(ts_dir, 'ts_cum_interp0_method1.csv')
ts_cum = np.genfromtxt(tscum_path)
cls.ts_incr = np.reshape(ts_incr,
newshape=cls.tsincr_0.shape,
order='F')
cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F')
def prepifg(config_file):
"""
Convert input files to geotiff and perform multilooking
(resampling) and/or cropping
"""
config_file = os.path.abspath(config_file)
params = cf.get_config_params(config_file)
log.info('This job was run with the following parameters:')
log.info(json.dumps(params, indent=4, sort_keys=True))
run_prepifg.main(params)
def prepifg(config_file):
"""
Convert input files to geotiff and perform multilooking
(resampling) and/or cropping.
"""
config_file = abspath(config_file)
params = cf.get_config_params(config_file)
if params[cf.LUIGI]:
run_prepifg.main()
else:
run_prepifg.main(params)
def test_spatio_temporal_filter():
import tempfile
from pyrate import shared
from os.path import basename, join
from collections import OrderedDict
from pyrate.scripts import run_prepifg
from osgeo import gdal
from pyrate import ifgconstants as ifc
ifg_out = sio.loadmat(
os.path.join(SML_TEST_DIR, 'matlab_aps',
'ifg_spatio_temp_out.mat'))['ifg']
tsincr = sio.loadmat(
os.path.join(SML_TEST_DIR, 'matlab_aps', 'tsincr_svd.mat'))['tsincr']
params = cf.get_config_params(os.path.join(TEST_CONF_GAMMA))
params[cf.OUT_DIR] = tempfile.mkdtemp()
params[cf.TMPDIR] = join(params[cf.OUT_DIR], cf.TMPDIR)
shared.mkdir_p(params[cf.TMPDIR])
params[cf.SLPF_METHOD] = 2
params[cf.SLPF_CUTOFF] = 0
params[cf.SLPF_ORDER] = 1
params[cf.SLPF_NANFILL] = 0
params[cf.TLPF_METHOD] = 3
params[cf.TLPF_CUTOFF] = 0.25
params[cf.TLPF_PTHR] = 5
ifgs = small_data_setup()
_ = [ifgs_pk.pop(k) for k in ['gt', 'epochlist', 'md', 'wkt']]
preread_ifgs = {
join(params[cf.OUT_DIR], basename(k)): v
for k, v in ifgs_pk.items()
}
preread_ifgs = OrderedDict(sorted(preread_ifgs.items()))
run_prepifg.main(params)
for k, v in preread_ifgs.items():
v.path = join(params[cf.OUT_DIR], basename(k))
preread_ifgs[k] = v
ifg = ifgs[0]
ifg.x_size = xpsize
ifg.y_size = ypsize
spatio_temporal_filter(tsincr, ifg, params, preread_ifgs)
for i in ifgs:
i.close()
for (p, v), i in zip(preread_ifgs.items(), range(ifg_out.shape[2])):
ds = gdal.Open(p)
metadata = ds.GetMetadata()
assert ifc.PYRATE_APS_ERROR in metadata
assert metadata[ifc.PYRATE_APS_ERROR] == ifc.APS_REMOVED
arr = ds.GetRasterBand(1).ReadAsArray()
np.testing.assert_array_almost_equal(arr, ifg_out[:, :, i], decimal=3)
def test_prepifg_mpi(mpisync, get_config, tempdir, roipac_or_gamma, get_lks,
get_crop):
from tests.common import TEST_CONF_ROIPAC, TEST_CONF_GAMMA
from os.path import join, basename
if roipac_or_gamma == 1:
params = get_config(TEST_CONF_GAMMA)
else:
params = get_config(TEST_CONF_ROIPAC)
outdir = mpiops.run_once(tempdir)
params[cf.OUT_DIR] = outdir
params[cf.PARALLEL] = False
params[cf.IFG_LKSX], params[cf.IFG_LKSY] = get_lks, get_lks
params[cf.IFG_CROP_OPT] = get_crop
if roipac_or_gamma == 1:
params[cf.IFG_FILE_LIST] = join(common.SML_TEST_GAMMA, 'ifms_17')
params[cf.OBS_DIR] = common.SML_TEST_GAMMA
params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
params[cf.DEM_HEADER_FILE] = common.SML_TEST_DEM_HDR_GAMMA
run_prepifg.main(params)
if mpiops.rank == 0:
if roipac_or_gamma == 1:
params_s = get_config(TEST_CONF_GAMMA)
else:
params_s = get_config(TEST_CONF_ROIPAC)
params_s[cf.OUT_DIR] = tempdir()
params_s[cf.PARALLEL] = True
params_s[cf.IFG_LKSX], params_s[cf.IFG_LKSY] = get_lks, get_lks
params_s[cf.IFG_CROP_OPT] = get_crop
if roipac_or_gamma == 1:
base_unw_paths = glob.glob(join(common.SML_TEST_GAMMA,
"*_utm.unw"))
run_prepifg.gamma_prepifg(base_unw_paths, params_s)
else:
base_unw_paths = glob.glob(join(common.SML_TEST_OBS, "*.unw"))
run_prepifg.roipac_prepifg(base_unw_paths, params_s)
mpi_tifs = glob.glob(join(outdir, "*.tif"))
serial_tifs = glob.glob(join(params[cf.OUT_DIR], "*.tif"))
mpi_tifs.sort()
serial_tifs.sort()
# 17 geotifs, and 17 mlooked tifs
assert len(mpi_tifs) == len(serial_tifs)
for m_f, s_f in zip(mpi_tifs, serial_tifs):
assert basename(m_f) == basename(s_f)
shutil.rmtree(outdir)
shutil.rmtree(params_s[cf.OUT_DIR])
def common_check(self, conf_file):
data_paths = glob.glob(os.path.join(SML_TEST_GAMMA, "*_utm.unw"))
self.make_input_files(data_paths)
base_ifg_paths, dest_paths, params = cf.get_ifg_paths(conf_file)
dest_base_ifgs = [
os.path.join(
params[cf.OUT_DIR],
os.path.basename(q).split('.')[0] + '_' +
os.path.basename(q).split('.')[1] + '.tif')
for q in base_ifg_paths
]
sys.argv = ['pyrate', 'prepifg', conf_file]
run_prepifg.main()
for p, q in zip(dest_base_ifgs, dest_paths):
self.assertTrue(os.path.exists(p), '{} does not exist'.format(p))
self.assertTrue(os.path.exists(q), '{} does not exist'.format(q))
def setUpClass(cls):
params = cf.get_config_params(TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['run_prepifg.py', TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.REF_EST_METHOD] = 2
run_prepifg.main(params)
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
ifgs = shared.pre_prepare_ifgs(dest_paths, params)
refx, refy = run_pyrate.ref_pixel_calc(dest_paths, params)
pyrate.orbital.remove_orbital_error(ifgs, params)
ifgs = prepare_ifgs_without_phase(dest_paths, params)
_, ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy)
# Calculate interferogram noise
cls.maxvar = [cvd(i, params)[0] for i in ifgs]
cls.vcmt = get_vcmt(ifgs, cls.maxvar)
def setUpClass(cls):
params = cf.get_config_params(common.TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['run_prepifg.py', common.TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
run_prepifg.main(params)
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.REF_EST_METHOD] = 2
params[cf.PARALLEL] = False
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop,
params, xlks)
# start run_pyrate copy
ifgs = shared.pre_prepare_ifgs(dest_paths, params)
mst_grid = tests.common.mst_calculation(dest_paths, params)
# Estimate reference pixel location
refx, refy = run_pyrate.ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.orbital.remove_orbital_error(ifgs, params)
for i in ifgs:
i.close()
ifgs = shared.pre_prepare_ifgs(dest_paths, params)
cls.ref_phs, cls.ifgs = estimate_ref_phase(ifgs, params, refx, refy)
# end run_pyrate copy
for i in ifgs:
i.close()
def setUpClass(cls):
params = cf.get_config_params(TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['run_prepifg.py', TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.TMPDIR] = os.path.join(cls.temp_out_dir, cf.TMPDIR)
shared.mkdir_p(params[cf.TMPDIR])
params[cf.REF_EST_METHOD] = 2
run_prepifg.main(params)
cls.params = params
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
ifgs = common.pre_prepare_ifgs(dest_paths, params)
refx, refy = run_pyrate._ref_pixel_calc(dest_paths, params)
pyrate.orbital.remove_orbital_error(ifgs, params)
ifgs = prepare_ifgs_without_phase(dest_paths, params)
_, cls.ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy)
r_dist = RDist(ifgs[0])()
# Calculate interferogram noise
cls.maxvar = [cvd(i, params, r_dist, calc_alpha=True,
save_acg=True, write_vals=True)[0] for i in ifgs]
cls.vcmt = get_vcmt(ifgs, cls.maxvar)
def setUpClass(cls):
params = cf.get_config_params(TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['run_prepifg.py', TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
shared.mkdir_p(params[cf.TMPDIR])
run_prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
xlks, _, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
# start run_pyrate copy
ifgs = pre_prepare_ifgs(dest_paths, params)
mst_grid = tests.common.mst_calculation(dest_paths, params)
refx, refy = run_pyrate._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.orbital.remove_orbital_error(ifgs, params)
ifgs = prepare_ifgs_without_phase(dest_paths, params)
_, ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy)
r_dist = vcm_module.RDist(ifgs[0])()
maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in ifgs]
vcmt = vcm_module.get_vcmt(ifgs, maxvar)
# Calculate linear rate map
params[cf.PARALLEL] = 1
cls.rate, cls.error, cls.samples = tests.common.calculate_linear_rate(
ifgs, params, vcmt, mst_mat=mst_grid)
params[cf.PARALLEL] = 2
cls.rate_2, cls.error_2, cls.samples_2 = \
tests.common.calculate_linear_rate(ifgs, params, vcmt,
mst_mat=mst_grid)
params[cf.PARALLEL] = 0
# Calculate linear rate map
cls.rate_s, cls.error_s, cls.samples_s = \
tests.common.calculate_linear_rate(ifgs, params, vcmt,
mst_mat=mst_grid)
matlab_linrate_dir = os.path.join(SML_TEST_DIR, 'matlab_linrate')
cls.rate_matlab = np.genfromtxt(os.path.join(matlab_linrate_dir,
'stackmap.csv'),
delimiter=',')
cls.error_matlab = np.genfromtxt(os.path.join(matlab_linrate_dir,
'errormap.csv'),
delimiter=',')
cls.samples_matlab = np.genfromtxt(os.path.join(
matlab_linrate_dir, 'coh_sta.csv'),
delimiter=',')
def setUpClass(cls):
cls.tif_dir_inc = tempfile.mkdtemp()
cls.tif_dir_ele = tempfile.mkdtemp()
cls.tif_dir_ele_par = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_ROIPAC
# change the required params
cls.params_inc = cf.get_config_params(cls.test_conf)
cls.params_inc[cf.OBS_DIR] = common.SML_TEST_GAMMA
cls.params_inc[cf.PROCESSOR] = 1 # gamma
file_list = cf.parse_namelist(
os.path.join(common.SML_TEST_GAMMA, 'ifms_17'))
# config file
cls.params_inc[cf.IFG_FILE_LIST] = tempfile.mktemp(dir=cls.tif_dir_inc)
# write a short filelist with only 3 gamma unws
with open(cls.params_inc[cf.IFG_FILE_LIST], 'w') as fp:
for f in file_list[:2]:
fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n')
cls.params_inc[cf.OUT_DIR] = cls.tif_dir_inc
cls.params_inc[cf.PARALLEL] = 0
cls.params_inc[cf.REF_EST_METHOD] = 1
cls.params_inc[cf.APS_METHOD] = 2
cls.params_inc[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
cls.params_inc[cf.APS_INCIDENCE_MAP] = common.SML_TEST_INCIDENCE
run_prepifg.main(cls.params_inc)
# now create the config for the elevation_map case
cls.params_ele = copy.copy(cls.params_inc)
cls.params_ele[cf.OUT_DIR] = cls.tif_dir_ele
cls.params_ele[cf.APS_METHOD] = 2
cls.params_ele[cf.APS_INCIDENCE_MAP] = None
cls.params_ele[cf.APS_INCIDENCE_EXT] = None
cls.params_ele[cf.APS_ELEVATION_MAP] = common.SML_TEST_ELEVATION
cls.params_ele[cf.APS_ELEVATION_EXT] = 'lv_theta'
run_prepifg.main(cls.params_ele)
ptn = re.compile(r'\d{8}')
dest_paths_inc = [
f for f in glob.glob(os.path.join(cls.tif_dir_inc, '*.tif'))
if ("cr" in f) and ("rlks" in f) and (
len(re.findall(ptn, os.path.basename(f))) == 2)
]
cls.ifgs_inc = common.small_data_setup(datafiles=dest_paths_inc)
dest_paths_ele = [
f for f in glob.glob(os.path.join(cls.tif_dir_ele, '*.tif'))
if "cr" in f and "rlks" in f and (
len(re.findall(ptn, os.path.basename(f))) == 2)
]
cls.ifgs_ele = common.small_data_setup(datafiles=dest_paths_ele)
# now create the config for the elevation map parallel case
cls.params_ele_par = copy.copy(cls.params_ele)
cls.params_ele_par[cf.OUT_DIR] = cls.tif_dir_ele_par
cls.params_ele_par[cf.PARALLEL] = True
run_prepifg.main(cls.params_ele_par)
dest_paths_ele_par = \
[f for f in glob.glob(os.path.join(cls.tif_dir_ele_par, '*.tif'))
if "cr" in f and "rlks" in f and
(len(re.findall(ptn, os.path.basename(f))) == 2)]
cls.ifgs_ele_par = common.small_data_setup(
datafiles=dest_paths_ele_par)
aps.remove_aps_delay(cls.ifgs_inc, cls.params_inc)
aps.remove_aps_delay(cls.ifgs_ele, cls.params_ele)
aps.remove_aps_delay(cls.ifgs_ele_par, cls.params_ele_par)
