def test_new_config_file_and_original_match(self):
params = config.get_config_params(TEST_CONF_GAMMA)
temp_config = tempfile.mktemp(suffix='.conf')
config.write_config_file(params, temp_config)
new_params = config.get_config_params(temp_config)
self.maxDiff = None
self.assertDictEqual(params, new_params)
os.remove(temp_config)
def setUpClass(cls):
# testing constants2
cls.BASE_DIR = tempfile.mkdtemp()
cls.BASE_OUT_DIR = join(cls.BASE_DIR, 'out')
cls.BASE_DEM_DIR = join(cls.BASE_DIR, 'dem')
cls.BASE_DEM_FILE = join(cls.BASE_DEM_DIR, 'roipac_test_trimmed.tif')
try:
# copy source data (treat as prepifg already run)
os.makedirs(cls.BASE_OUT_DIR)
for path in glob.glob(join(common.SML_TEST_TIF, '*')):
dest = join(cls.BASE_OUT_DIR, os.path.basename(path))
shutil.copy(path, dest)
os.chmod(dest, 0o660)
os.makedirs(cls.BASE_DEM_DIR)
orig_dem = common.SML_TEST_DEM_TIF
os.symlink(orig_dem, cls.BASE_DEM_FILE)
os.chdir(cls.BASE_DIR)
params = config.get_config_params(common.TEST_CONF_ROIPAC)
params[cf.OUT_DIR] = cls.BASE_OUT_DIR
params[cf.PROCESSOR] = 0 # roipac
params[cf.APS_CORRECTION] = 0
paths = glob.glob(join(cls.BASE_OUT_DIR, 'geo_*-*.tif'))
params[cf.PARALLEL] = False
run_pyrate.process_ifgs(sorted(paths), params, 2, 2)
if not hasattr(cls, 'ifgs'):
cls.ifgs = get_ifgs(out_dir=cls.BASE_OUT_DIR)
except:
# revert working dir & avoid paths busting other tests
os.chdir(CURRENT_DIR)
raise
def common_check(self, ele, inc):
os.path.exists(self.conf_file)
params = cf.get_config_params(self.conf_file)
sys.argv = ['dummy', self.conf_file]
run_prepifg.main(params)
# test 17 geotiffs created
geotifs = glob.glob(os.path.join(self.base_dir, '*_unw.tif'))
self.assertEqual(17, len(geotifs))
# test dem geotiff created
demtif = glob.glob(os.path.join(self.base_dir, '*_dem.tif'))
self.assertEqual(1, len(demtif))
# elevation/incidence file
ele = glob.glob(
os.path.join(self.base_dir, '*utm_{ele}.tif'.format(ele=ele)))[0]
self.assertTrue(os.path.exists(ele))
# mlooked tifs
mlooked_tifs = [
f for f in glob.glob(os.path.join(self.base_dir, '*.tif'))
if "cr" in f and "rlks" in f
]
# 19 including 17 ifgs, 1 dem and one incidence
self.assertEqual(19, len(mlooked_tifs))
inc = glob.glob(
os.path.join(self.base_dir, '*utm_{inc}.tif'.format(inc=inc)))
self.assertEqual(0, len(inc))
def test_read_param_file_missing_value():
# ensure the parser can handle blank option values
conf_path = join(SML_TEST_CONF, 'pyrate2.conf')
params = config.get_config_params(conf_path)
assert params[config.REFX] == -1
assert params[config.REFY] == -1
def setUp(self):
self.BASE_DIR = tempfile.mkdtemp()
self.params = cf.get_config_params(TEST_CONF_ROIPAC)
# change to orbital error correction method 2
self.params[cf.ORBITAL_FIT_METHOD] = NETWORK_METHOD
self.params[cf.ORBITAL_FIT_LOOKS_X] = 1
self.params[cf.ORBITAL_FIT_LOOKS_Y] = 1
data_paths = [
os.path.join(SML_TEST_TIF, p) for p in small_ifg_file_list()
]
self.new_data_paths = [
os.path.join(self.BASE_DIR, os.path.basename(d))
for d in data_paths
]
for d in data_paths:
d_copy = os.path.join(self.BASE_DIR, os.path.basename(d))
shutil.copy(d, d_copy)
os.chmod(d_copy, 0o660)
self.ifgs = small_data_setup(datafiles=self.new_data_paths)
for i in self.ifgs:
if not i.is_open:
i.open()
if not i.nan_converted:
i.convert_to_nans()
if not i.mm_converted:
i.convert_to_mm()
i.write_modified_phase()
def test_read_param_file_missing_option():
# ensure the parser can handle missing option fields
conf_path = join(SML_TEST_CONF, 'pyrate1.conf')
params = config.get_config_params(conf_path)
assert params[config.REFX] == -1
assert params[config.REFY] == -1
def setUpClass(cls):
cls.tif_dir = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_GAMMA
# change the required params
cls.params = cf.get_config_params(cls.test_conf)
cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA
cls.params[cf.PROCESSOR] = 1 # gamma
file_list = list(cf.parse_namelist(os.path.join(common.SML_TEST_GAMMA,
'ifms_17')))
fd, cls.params[cf.IFG_FILE_LIST] = tempfile.mkstemp(suffix='.conf',
dir=cls.tif_dir)
os.close(fd)
# write a short filelist with only 3 gamma unws
with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp:
for f in file_list[:3]:
fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n')
cls.params[cf.OUT_DIR] = cls.tif_dir
cls.params[cf.PARALLEL] = 0
cls.params[cf.REF_EST_METHOD] = 1
cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
# base_unw_paths need to be geotiffed and multilooked by run_prepifg
cls.base_unw_paths = cf.original_ifg_paths(
cls.params[cf.IFG_FILE_LIST])
cls.base_unw_paths.append(common.SML_TEST_DEM_GAMMA)
xlks, ylks, crop = cf.transform_params(cls.params)
# dest_paths are tifs that have been geotif converted and multilooked
run_prepifg.gamma_prepifg(cls.base_unw_paths, cls.params)
cls.base_unw_paths.pop() # removed dem as we don't want it in ifgs
dest_paths = cf.get_dest_paths(
cls.base_unw_paths, crop, cls.params, xlks)
cls.ifgs = common.small_data_setup(datafiles=dest_paths)
def setUp(self):
self.ifgs = small_data_setup()
self.params = cf.get_config_params(TEST_CONF_ROIPAC)
self.params[cf.REFNX] = REFNX
self.params[cf.REFNY] = REFNY
self.params[cf.REF_CHIP_SIZE] = CHIPSIZE
self.params[cf.REF_MIN_FRAC] = MIN_FRAC
self.params[cf.PARALLEL] = PARALLEL
def setUpClass(cls):
params = cf.get_config_params(common.TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['run_prepifg.py', common.TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
run_prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
# start run_pyrate copy
ifgs = common.pre_prepare_ifgs(dest_paths, params)
mst_grid = common.mst_calculation(dest_paths, params)
refx, refy = run_pyrate._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.orbital.remove_orbital_error(ifgs, params)
ifgs = common.prepare_ifgs_without_phase(dest_paths, params)
_, ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy)
r_dist = covariance.RDist(ifgs[0])()
maxvar = [covariance.cvd(i, params, r_dist)[0] for i in ifgs]
vcmt = covariance.get_vcmt(ifgs, maxvar)
params[cf.TIME_SERIES_METHOD] = 1
params[cf.PARALLEL] = 0
# Calculate time series
cls.tsincr_0, cls.tscum_0, _ = common.calculate_time_series(
ifgs, params, vcmt, mst=mst_grid)
params[cf.PARALLEL] = 1
cls.tsincr_1, cls.tscum_1, cls.tsvel_1 = \
common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid)
params[cf.PARALLEL] = 2
cls.tsincr_2, cls.tscum_2, cls.tsvel_2 = \
common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid)
# load the matlab data
ts_dir = os.path.join(common.SML_TEST_DIR, 'matlab_time_series')
tsincr_path = os.path.join(ts_dir, 'ts_incr_interp0_method1.csv')
ts_incr = np.genfromtxt(tsincr_path)
# the matlab tsvel return is a bit pointless and not tested here
# tserror is not returned
# tserr_path = os.path.join(SML_TIME_SERIES_DIR,
# 'ts_error_interp0_method1.csv')
# ts_err = np.genfromtxt(tserr_path, delimiter=',')
tscum_path = os.path.join(ts_dir, 'ts_cum_interp0_method1.csv')
ts_cum = np.genfromtxt(tscum_path)
cls.ts_incr = np.reshape(ts_incr,
newshape=cls.tsincr_0.shape,
order='F')
cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F')
def prepifg(config_file):
"""
Convert input files to geotiff and perform multilooking
(resampling) and/or cropping
"""
config_file = os.path.abspath(config_file)
params = cf.get_config_params(config_file)
log.info('This job was run with the following parameters:')
log.info(json.dumps(params, indent=4, sort_keys=True))
run_prepifg.main(params)
def setUpClass(cls):
cls.tif_dir = tempfile.mkdtemp()
cls.tif_dir_method2 = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_ROIPAC
# change the required params
cls.params = cf.get_config_params(cls.test_conf)
cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA
cls.params[cf.PROCESSOR] = 1 # gamma
file_list = cf.parse_namelist(
os.path.join(common.SML_TEST_GAMMA, 'ifms_17'))
cls.params[cf.IFG_FILE_LIST] = tempfile.mktemp(dir=cls.tif_dir)
# write a short filelist with only 3 gamma unws
with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp:
for f in file_list[:2]:
fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n')
cls.params[cf.OUT_DIR] = cls.tif_dir
cls.params[cf.PARALLEL] = True
cls.params[cf.REF_EST_METHOD] = 1
cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
cls.params[cf.APS_INCIDENCE_MAP] = common.SML_TEST_INCIDENCE
# base_unw_paths need to be geotiffed and multilooked by run_prepifg
base_unw_paths = run_pyrate.original_ifg_paths(
cls.params[cf.IFG_FILE_LIST])
# add dem
base_unw_paths.append(common.SML_TEST_DEM_GAMMA)
# add incidence
base_unw_paths.append(common.SML_TEST_INCIDENCE)
xlks, ylks, crop = run_pyrate.transform_params(cls.params)
import copy
cls.params_method2 = copy.copy(cls.params)
cls.params_method2[cf.OUT_DIR] = cls.tif_dir_method2
cls.params_method2[cf.APS_METHOD] = 2
# dest_paths are tifs that have been geotif converted and multilooked
run_prepifg.gamma_prepifg(base_unw_paths, cls.params)
run_prepifg.gamma_prepifg(base_unw_paths, cls.params_method2)
# removed incidence as we don't want it in ifgs list
base_unw_paths.pop()
# removed dem as we don't want it in ifgs list
base_unw_paths.pop()
dest_paths = run_pyrate.get_dest_paths(base_unw_paths, crop,
cls.params, xlks)
cls.ifgs = common.small_data_setup(datafiles=dest_paths)
dest_paths_m2 = run_pyrate.get_dest_paths(base_unw_paths, crop,
cls.params_method2, xlks)
cls.ifgs_method2 = common.small_data_setup(datafiles=dest_paths_m2)
aps.remove_aps_delay(cls.ifgs, cls.params)
aps.remove_aps_delay(cls.ifgs_method2, cls.params_method2)
def prepifg(config_file):
"""
Convert input files to geotiff and perform multilooking
(resampling) and/or cropping.
"""
config_file = abspath(config_file)
params = cf.get_config_params(config_file)
if params[cf.LUIGI]:
run_prepifg.main()
else:
run_prepifg.main(params)
def setUpClass(cls):
rate_types = ['linrate', 'linerror', 'linsamples']
cls.tif_dir = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_GAMMA
# change the required params
params = cf.get_config_params(cls.test_conf)
params[cf.OBS_DIR] = common.SML_TEST_GAMMA
params[cf.PROCESSOR] = 1 # gamma
params[cf.IFG_FILE_LIST] = os.path.join(common.SML_TEST_GAMMA,
'ifms_17')
params[cf.OUT_DIR] = cls.tif_dir
params[cf.PARALLEL] = 0
params[cf.APS_CORRECTION] = False
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
xlks, ylks, crop = cf.transform_params(params)
# base_unw_paths need to be geotiffed and multilooked by run_prepifg
base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
# dest_paths are tifs that have been geotif converted and multilooked
cls.dest_paths = cf.get_dest_paths(base_unw_paths, crop, params, xlks)
run_prepifg.gamma_prepifg(base_unw_paths, params)
tiles = run_pyrate.get_tiles(cls.dest_paths[0], 3, 3)
ifgs = common.small_data_setup()
cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = \
run_pyrate.process_ifgs(cls.dest_paths, params, 3, 3)
cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate_p, cls.error_p, cls.samples_p = [
common.reconstruct_linrate(ifgs[0].shape, tiles, params[cf.TMPDIR],
t) for t in rate_types
]
# now create the non parallel version
cls.tif_dir_s = tempfile.mkdtemp()
params[cf.PARALLEL] = 0
params[cf.OUT_DIR] = cls.tif_dir_s
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
cls.dest_paths_s = cf.get_dest_paths(base_unw_paths, crop, params,
xlks)
run_prepifg.gamma_prepifg(base_unw_paths, params)
cls.refpixel, cls.maxvar, cls.vcmt = \
run_pyrate.process_ifgs(cls.dest_paths_s, params, 3, 3)
cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate, cls.error, cls.samples = [
common.reconstruct_linrate(ifgs[0].shape, tiles, params[cf.TMPDIR],
t) for t in rate_types
]
def test_spatio_temporal_filter():
import tempfile
from pyrate import shared
from os.path import basename, join
from collections import OrderedDict
from pyrate.scripts import run_prepifg
from osgeo import gdal
from pyrate import ifgconstants as ifc
ifg_out = sio.loadmat(
os.path.join(SML_TEST_DIR, 'matlab_aps',
'ifg_spatio_temp_out.mat'))['ifg']
tsincr = sio.loadmat(
os.path.join(SML_TEST_DIR, 'matlab_aps', 'tsincr_svd.mat'))['tsincr']
params = cf.get_config_params(os.path.join(TEST_CONF_GAMMA))
params[cf.OUT_DIR] = tempfile.mkdtemp()
params[cf.TMPDIR] = join(params[cf.OUT_DIR], cf.TMPDIR)
shared.mkdir_p(params[cf.TMPDIR])
params[cf.SLPF_METHOD] = 2
params[cf.SLPF_CUTOFF] = 0
params[cf.SLPF_ORDER] = 1
params[cf.SLPF_NANFILL] = 0
params[cf.TLPF_METHOD] = 3
params[cf.TLPF_CUTOFF] = 0.25
params[cf.TLPF_PTHR] = 5
ifgs = small_data_setup()
_ = [ifgs_pk.pop(k) for k in ['gt', 'epochlist', 'md', 'wkt']]
preread_ifgs = {
join(params[cf.OUT_DIR], basename(k)): v
for k, v in ifgs_pk.items()
}
preread_ifgs = OrderedDict(sorted(preread_ifgs.items()))
run_prepifg.main(params)
for k, v in preread_ifgs.items():
v.path = join(params[cf.OUT_DIR], basename(k))
preread_ifgs[k] = v
ifg = ifgs[0]
ifg.x_size = xpsize
ifg.y_size = ypsize
spatio_temporal_filter(tsincr, ifg, params, preread_ifgs)
for i in ifgs:
i.close()
for (p, v), i in zip(preread_ifgs.items(), range(ifg_out.shape[2])):
ds = gdal.Open(p)
metadata = ds.GetMetadata()
assert ifc.PYRATE_APS_ERROR in metadata
assert metadata[ifc.PYRATE_APS_ERROR] == ifc.APS_REMOVED
arr = ds.GetRasterBand(1).ReadAsArray()
np.testing.assert_array_almost_equal(arr, ifg_out[:, :, i], decimal=3)
def test_inc_vs_ele_maps_ele_provided(self):
self.make_input_files(ele=common.SML_TEST_ELEVATION)
assert os.path.exists(self.conf_file)
params = config.get_config_params(self.conf_file)
# incidence variables
self.assertIn(config.APS_INCIDENCE_MAP, params.keys())
self.assertIn(config.APS_INCIDENCE_EXT, params.keys())
self.assertIsNone(params[config.APS_INCIDENCE_MAP])
self.assertIsNone(params[config.APS_INCIDENCE_EXT])
# elevation variables
self.assertIn(config.APS_ELEVATION_MAP, params.keys())
self.assertIsNotNone(params[config.APS_ELEVATION_MAP])
self.assertIn(config.APS_ELEVATION_EXT, params.keys())
self.assertIn(config.APS_ELEVATION_MAP, params.keys())
def setUp(self):
self.ifg_paths = small_ifg_file_list()
self.params = cf.get_config_params(TEST_CONF_ROIPAC)
self.params[cf.PARALLEL] = False
self.params[cf.OUT_DIR] = tempfile.mkdtemp()
self.params_alt_ref_frac = copy.copy(self.params)
self.params_alt_ref_frac[cf.REF_MIN_FRAC] = 0.5
self.params_all_2s = copy.copy(self.params)
self.params_all_2s[cf.REFNX] = 2
self.params_all_2s[cf.REFNY] = 2
self.params_chipsize_15 = copy.copy(self.params_all_2s)
self.params_chipsize_15[cf.REF_CHIP_SIZE] = 15
self.params_all_1s = copy.copy(self.params)
self.params_all_1s[cf.REFNX] = 1
self.params_all_1s[cf.REFNY] = 1
self.params_all_1s[cf.REF_MIN_FRAC] = 0.7
def setUp(self):
self.BASE_DIR = tempfile.mkdtemp()
self.params = cf.get_config_params(TEST_CONF_ROIPAC)
# change to orbital error correction method 1
self.params[cf.ORBITAL_FIT_METHOD] = INDEPENDENT_METHOD
self.params[cf.ORBITAL_FIT_LOOKS_X] = 2
self.params[cf.ORBITAL_FIT_LOOKS_Y] = 2
self.params[cf.PARALLEL] = False
data_paths = [os.path.join(SML_TEST_TIF, p) for p in IFMS16]
self.ifg_paths = [
os.path.join(self.BASE_DIR, os.path.basename(d))
for d in data_paths
]
for d in data_paths:
shutil.copy(d, os.path.join(self.BASE_DIR, os.path.basename(d)))
def setUpClass(cls):
params = cf.get_config_params(TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['run_prepifg.py', TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.REF_EST_METHOD] = 2
run_prepifg.main(params)
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
ifgs = shared.pre_prepare_ifgs(dest_paths, params)
refx, refy = run_pyrate.ref_pixel_calc(dest_paths, params)
pyrate.orbital.remove_orbital_error(ifgs, params)
ifgs = prepare_ifgs_without_phase(dest_paths, params)
_, ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy)
# Calculate interferogram noise
cls.maxvar = [cvd(i, params)[0] for i in ifgs]
cls.vcmt = get_vcmt(ifgs, cls.maxvar)
def setUpClass(cls):
params = cf.get_config_params(common.TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['run_prepifg.py', common.TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
run_prepifg.main(params)
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.REF_EST_METHOD] = 2
params[cf.PARALLEL] = False
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop,
params, xlks)
# start run_pyrate copy
ifgs = shared.pre_prepare_ifgs(dest_paths, params)
mst_grid = tests.common.mst_calculation(dest_paths, params)
# Estimate reference pixel location
refx, refy = run_pyrate.ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.orbital.remove_orbital_error(ifgs, params)
for i in ifgs:
i.close()
ifgs = shared.pre_prepare_ifgs(dest_paths, params)
cls.ref_phs, cls.ifgs = estimate_ref_phase(ifgs, params, refx, refy)
# end run_pyrate copy
for i in ifgs:
i.close()
def setUpClass(cls):
params = config.get_config_params(common.TEST_CONF_ROIPAC)
cls.tmp_dir = tempfile.mkdtemp()
shared.copytree(common.SML_TEST_TIF, cls.tmp_dir)
tifs = glob.glob(os.path.join(cls.tmp_dir, "*.tif"))
for t in tifs:
os.chmod(t, 0o644)
small_ifgs = common.small_data_setup(datafiles=tifs)
ifg_paths = [i.data_path for i in small_ifgs]
cls.ifg_ret = shared.pre_prepare_ifgs(ifg_paths, params=params)
for i in cls.ifg_ret:
i.close()
nan_conversion = params[cf.NAN_CONVERSION]
# prepare a second set
cls.tmp_dir2 = tempfile.mkdtemp()
shared.copytree(common.SML_TEST_TIF, cls.tmp_dir2)
tifs = glob.glob(os.path.join(cls.tmp_dir2, "*.tif"))
for t in tifs:
os.chmod(t, 0o644)
small_ifgs = common.small_data_setup(datafiles=tifs)
ifg_paths = [i.data_path for i in small_ifgs]
cls.ifgs = [shared.Ifg(p) for p in ifg_paths]
for i in cls.ifgs:
if not i.is_open:
i.open(readonly=False)
if nan_conversion: # nan conversion happens here in networkx mst
i.nodata_value = params[cf.NO_DATA_VALUE]
i.convert_to_nans()
if not i.mm_converted:
i.convert_to_mm()
i.close()
def setUpClass(cls):
params = cf.get_config_params(TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['run_prepifg.py', TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.TMPDIR] = os.path.join(cls.temp_out_dir, cf.TMPDIR)
shared.mkdir_p(params[cf.TMPDIR])
params[cf.REF_EST_METHOD] = 2
run_prepifg.main(params)
cls.params = params
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
ifgs = common.pre_prepare_ifgs(dest_paths, params)
refx, refy = run_pyrate._ref_pixel_calc(dest_paths, params)
pyrate.orbital.remove_orbital_error(ifgs, params)
ifgs = prepare_ifgs_without_phase(dest_paths, params)
_, cls.ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy)
r_dist = RDist(ifgs[0])()
# Calculate interferogram noise
cls.maxvar = [cvd(i, params, r_dist, calc_alpha=True,
save_acg=True, write_vals=True)[0] for i in ifgs]
cls.vcmt = get_vcmt(ifgs, cls.maxvar)
# tsincr matrix from matlab using svd timeseries method
tsincr_svd = sio.loadmat(
os.path.join(SML_TEST_DIR, 'matlab_aps', 'tsincr_svd.mat'))
# prepread ifgs pickle file
preread_pk = 'preread_ifgs.pk' if PY3 else 'preread_ifgs_py2.pk'
ifgs_pk = os.path.join(SML_TEST_DIR, 'matlab_aps', preread_pk)
ifgs_pk = pickle.load(open(ifgs_pk, 'rb'))
# tlp filter output from matlab
ts_hp = sio.loadmat(os.path.join(SML_TEST_DIR, 'matlab_aps', 'ts_hp.mat'))
ts_hp_m2 = sio.loadmat(os.path.join(SML_TEST_DIR, 'matlab_aps',
'ts_hp_m2.mat'))
ts_hp_m3 = sio.loadmat(os.path.join(SML_TEST_DIR, 'matlab_aps',
'ts_hp_m3.mat'))
epochlist = get_epochs(ifgs_pk)[0]
params = cf.get_config_params(os.path.join(TEST_CONF_GAMMA))
@pytest.fixture(params=[1, 2, 3])
def tlpfilter_method(request):
return request.param
@pytest.fixture(params=[1, 2])
def slpfilter_method(request):
return request.param
def test_tlpfilter_matlab(tlpfilter_method):
params[cf.TLPF_METHOD] = tlpfilter_method
ts = ts_hp if tlpfilter_method == 1 \
def params(conf_file):
return cf.get_config_params(conf_file)
def setUp(self):
self.ifgs = small_data_setup()
self.params = cf.get_config_params(common.TEST_CONF_ROIPAC)
def setUpClass(cls):
cls.tif_dir_inc = tempfile.mkdtemp()
cls.tif_dir_ele = tempfile.mkdtemp()
cls.tif_dir_ele_par = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_ROIPAC
# change the required params
cls.params_inc = cf.get_config_params(cls.test_conf)
cls.params_inc[cf.OBS_DIR] = common.SML_TEST_GAMMA
cls.params_inc[cf.PROCESSOR] = 1 # gamma
file_list = cf.parse_namelist(
os.path.join(common.SML_TEST_GAMMA, 'ifms_17'))
# config file
cls.params_inc[cf.IFG_FILE_LIST] = tempfile.mktemp(dir=cls.tif_dir_inc)
# write a short filelist with only 3 gamma unws
with open(cls.params_inc[cf.IFG_FILE_LIST], 'w') as fp:
for f in file_list[:2]:
fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n')
cls.params_inc[cf.OUT_DIR] = cls.tif_dir_inc
cls.params_inc[cf.PARALLEL] = 0
cls.params_inc[cf.REF_EST_METHOD] = 1
cls.params_inc[cf.APS_METHOD] = 2
cls.params_inc[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
cls.params_inc[cf.APS_INCIDENCE_MAP] = common.SML_TEST_INCIDENCE
run_prepifg.main(cls.params_inc)
# now create the config for the elevation_map case
cls.params_ele = copy.copy(cls.params_inc)
cls.params_ele[cf.OUT_DIR] = cls.tif_dir_ele
cls.params_ele[cf.APS_METHOD] = 2
cls.params_ele[cf.APS_INCIDENCE_MAP] = None
cls.params_ele[cf.APS_INCIDENCE_EXT] = None
cls.params_ele[cf.APS_ELEVATION_MAP] = common.SML_TEST_ELEVATION
cls.params_ele[cf.APS_ELEVATION_EXT] = 'lv_theta'
run_prepifg.main(cls.params_ele)
ptn = re.compile(r'\d{8}')
dest_paths_inc = [
f for f in glob.glob(os.path.join(cls.tif_dir_inc, '*.tif'))
if ("cr" in f) and ("rlks" in f) and (
len(re.findall(ptn, os.path.basename(f))) == 2)
]
cls.ifgs_inc = common.small_data_setup(datafiles=dest_paths_inc)
dest_paths_ele = [
f for f in glob.glob(os.path.join(cls.tif_dir_ele, '*.tif'))
if "cr" in f and "rlks" in f and (
len(re.findall(ptn, os.path.basename(f))) == 2)
]
cls.ifgs_ele = common.small_data_setup(datafiles=dest_paths_ele)
# now create the config for the elevation map parallel case
cls.params_ele_par = copy.copy(cls.params_ele)
cls.params_ele_par[cf.OUT_DIR] = cls.tif_dir_ele_par
cls.params_ele_par[cf.PARALLEL] = True
run_prepifg.main(cls.params_ele_par)
dest_paths_ele_par = \
[f for f in glob.glob(os.path.join(cls.tif_dir_ele_par, '*.tif'))
if "cr" in f and "rlks" in f and
(len(re.findall(ptn, os.path.basename(f))) == 2)]
cls.ifgs_ele_par = common.small_data_setup(
datafiles=dest_paths_ele_par)
aps.remove_aps_delay(cls.ifgs_inc, cls.params_inc)
aps.remove_aps_delay(cls.ifgs_ele, cls.params_ele)
aps.remove_aps_delay(cls.ifgs_ele_par, cls.params_ele_par)
def setUpClass(cls):
cls.tif_dir_serial = tempfile.mkdtemp()
cls.tif_dir_mpi = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_ROIPAC
# change the required params
cls.params = cf.get_config_params(cls.test_conf)
cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA
cls.params[cf.PROCESSOR] = 1 # gamma
file_list = cf.parse_namelist(
os.path.join(common.SML_TEST_GAMMA, 'ifms_17'))
cls.params[cf.IFG_FILE_LIST] = tempfile.mktemp(dir=cls.tif_dir_serial)
# write a short filelist with only 3 gamma unws
with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp:
for f in file_list[:2]:
fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n')
cls.params[cf.OUT_DIR] = cls.tif_dir_serial
cls.params[cf.PARALLEL] = 0
cls.params[cf.REF_EST_METHOD] = 2
cls.params[cf.IFG_LKSX] = 1
cls.params[cf.IFG_LKSY] = 1
cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
cls.params[cf.APS_INCIDENCE_MAP] = common.SML_TEST_INCIDENCE
# base_unw_paths need to be geotiffed and multilooked by run_prepifg
base_unw_paths = run_pyrate.original_ifg_paths(
cls.params[cf.IFG_FILE_LIST])
# add dem
base_unw_paths.append(common.SML_TEST_DEM_GAMMA)
# add incidence
base_unw_paths.append(common.SML_TEST_INCIDENCE)
xlks, ylks, crop = run_pyrate.transform_params(cls.params)
cls.params_mpi = copy.copy(cls.params)
cls.params_mpi[cf.OUT_DIR] = cls.tif_dir_mpi
# dest_paths are tifs that have been geotif converted and multilooked
run_prepifg.gamma_prepifg(base_unw_paths, cls.params)
run_prepifg.gamma_prepifg(base_unw_paths, cls.params_mpi)
# removed incidence as we don't want it in ifgs list
base_unw_paths.pop()
# removed dem as we don't want it in ifgs list
base_unw_paths.pop()
dest_paths = run_pyrate.get_dest_paths(base_unw_paths, crop,
cls.params, xlks)
dest_paths_mpi = run_pyrate.get_dest_paths(base_unw_paths, crop,
cls.params_mpi, xlks)
run_pyrate.process_ifgs(dest_paths, cls.params)
cls.ifgs_serial = common.small_data_setup(datafiles=dest_paths)
cls.conf_mpi = tempfile.mktemp('.conf', dir=cls.tif_dir_mpi)
cf.write_config_file(cls.params_mpi, cls.conf_mpi)
str = 'mpirun -np 4 python pyrate/nci/run_pyrate_pypar.py ' + \
cls.conf_mpi
cmd = str.split()
subprocess.check_call(cmd)
str = 'mpirun -np 4 python pyrate/nci/run_pyrate_pypar_2.py ' + \
cls.conf_mpi
cmd = str.split()
subprocess.check_call(cmd)
str = 'mpirun -np 4 python pyrate/nci/run_pyrate_pypar_3.py ' + \
cls.conf_mpi
cmd = str.split()
subprocess.check_call(cmd)
cls.ifgs_mpi = common.small_data_setup(datafiles=dest_paths_mpi)
def test_read_param_file():
params = config.get_config_params(TEST_CONF_ROIPAC)
for k in params.keys():
assert k and len(k) > 1
assert params[k] != ''
assert not k.endswith(":") # are the colons removed?
def test_write_config_file(self):
params = config.get_config_params(TEST_CONF_GAMMA)
temp_config = tempfile.mktemp(suffix='.conf')
config.write_config_file(params, temp_config)
self.assertTrue(os.path.exists(temp_config))
os.remove(temp_config)
def setUpClass(cls):
params = cf.get_config_params(TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['run_prepifg.py', TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
shared.mkdir_p(params[cf.TMPDIR])
run_prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
xlks, _, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
# start run_pyrate copy
ifgs = pre_prepare_ifgs(dest_paths, params)
mst_grid = tests.common.mst_calculation(dest_paths, params)
refx, refy = run_pyrate._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.orbital.remove_orbital_error(ifgs, params)
ifgs = prepare_ifgs_without_phase(dest_paths, params)
_, ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy)
r_dist = vcm_module.RDist(ifgs[0])()
maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in ifgs]
vcmt = vcm_module.get_vcmt(ifgs, maxvar)
# Calculate linear rate map
params[cf.PARALLEL] = 1
cls.rate, cls.error, cls.samples = tests.common.calculate_linear_rate(
ifgs, params, vcmt, mst_mat=mst_grid)
params[cf.PARALLEL] = 2
cls.rate_2, cls.error_2, cls.samples_2 = \
tests.common.calculate_linear_rate(ifgs, params, vcmt,
mst_mat=mst_grid)
params[cf.PARALLEL] = 0
# Calculate linear rate map
cls.rate_s, cls.error_s, cls.samples_s = \
tests.common.calculate_linear_rate(ifgs, params, vcmt,
mst_mat=mst_grid)
matlab_linrate_dir = os.path.join(SML_TEST_DIR, 'matlab_linrate')
cls.rate_matlab = np.genfromtxt(os.path.join(matlab_linrate_dir,
'stackmap.csv'),
delimiter=',')
cls.error_matlab = np.genfromtxt(os.path.join(matlab_linrate_dir,
'errormap.csv'),
delimiter=',')
cls.samples_matlab = np.genfromtxt(os.path.join(
matlab_linrate_dir, 'coh_sta.csv'),
delimiter=',')
def setUp(self):
self.conf_path = TEST_CONF_ROIPAC
self.params = config.get_config_params(self.conf_path)
