def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
self.params = dict()
self.params[cf.REF_EST_METHOD] = 1
self.params[cf.PARALLEL] = False
self.params[cf.TMPDIR] = self.tmp_dir
self.refpx, self.refpy = 38, 58
common.copytree(common.SML_TEST_TIF, self.tmp_dir)
small_tifs = glob.glob(os.path.join(self.tmp_dir, "*.tif"))
for s in small_tifs:
os.chmod(s, 0o644)
self.ifgs = common.small_data_setup(self.tmp_dir, is_dir=True)
for ifg in self.ifgs:
ifg.close()
def __assert_same_files_produced(dir1, dir2, ext, num_files):
dir1_files = list(Path(dir1).glob(ext))
dir2_files = list(Path(dir2).glob(ext))
dir1_files.sort()
dir2_files.sort()
# 17 unwrapped geotifs
# 17 cropped multilooked tifs + 1 dem
assert len(dir1_files) == num_files
assert len(dir2_files) == num_files
c = 0
all_roipac_ifgs = [f for f in small_data_setup(dir1_files) if not isinstance(f, DEM)]
all_gamma_ifgs = [f for f in small_data_setup(dir2_files) if not isinstance(f, DEM)]
for c, (i, j) in enumerate(zip(all_roipac_ifgs, all_gamma_ifgs)):
mdi = i.meta_data
mdj = j.meta_data
for k in mdi: # all key values equal
if k == "INCIDENCE_DEGREES":
pass # incidence angle not implemented for roipac
elif _is_number(mdi[k]):
assert pytest.approx(float(mdj[k]), 0.00001) == float(mdi[k])
elif mdi[k] == "ROIPAC" or "GAMMA":
pass # INSAR_PROCESSOR can not be equal
else:
assert mdj[k] == mdi[k]
if i.data_path.__contains__("_{looks}rlks_{crop}cr".format(looks=1, crop=1)):
# these are multilooked tifs
# test that DATA_STEP is MULTILOOKED
assert mdi[ifc.DATA_TYPE] == ifc.MULTILOOKED
assert mdj[ifc.DATA_TYPE] == ifc.MULTILOOKED
else:
assert mdi[ifc.DATA_TYPE] == ifc.ORIG
assert mdj[ifc.DATA_TYPE] == ifc.ORIG
assert c + 1 == len(all_gamma_ifgs)
def setUpClass(cls):
params = config.get_config_params(common.TEST_CONF_ROIPAC)
cls.tmp_dir = tempfile.mkdtemp()
common.copytree(common.SML_TEST_TIF, cls.tmp_dir)
tifs = glob.glob(os.path.join(cls.tmp_dir, "*.tif"))
for t in tifs:
os.chmod(t, 0o644)
small_ifgs = common.small_data_setup(datafiles=tifs)
ifg_paths = [i.data_path for i in small_ifgs]
cls.ifg_ret = common.pre_prepare_ifgs(ifg_paths, params=params)
for i in cls.ifg_ret:
i.close()
nan_conversion = params[cf.NAN_CONVERSION]
# prepare a second set
cls.tmp_dir2 = tempfile.mkdtemp()
common.copytree(common.SML_TEST_TIF, cls.tmp_dir2)
tifs = glob.glob(os.path.join(cls.tmp_dir2, "*.tif"))
for t in tifs:
os.chmod(t, 0o644)
small_ifgs = common.small_data_setup(datafiles=tifs)
ifg_paths = [i.data_path for i in small_ifgs]
cls.ifgs = [shared.Ifg(p) for p in ifg_paths]
for i in cls.ifgs:
if not i.is_open:
i.open(readonly=False)
if nan_conversion: # nan conversion happens here in networkx mst
i.nodata_value = params[cf.NO_DATA_VALUE]
i.convert_to_nans()
if not i.mm_converted:
i.convert_to_mm()
i.close()
def test_slpfilter_accumulated(slpfilter_method):
ts_aps_before = copy.copy(ts_hp_before_slpfilter)
params[cf.SLPF_METHOD] = slpfilter_method
ifgs = small_data_setup()
ts_aps_m = ts_aps_m1 if slpfilter_method == 1 else ts_aps_m2
Ifg = namedtuple('Ifg', 'x_centre, y_centre, x_size, y_size, shape')
ifg = Ifg(x_size=xpsize,
y_size=ypsize,
shape=ifgs[0].shape,
x_centre=ifgs[0].x_centre,
y_centre=ifgs[0].y_centre)
ts_aps = spatial_low_pass_filter(ts_aps_before, ifg, params=params)
for i in ifgs:
i.close()
np.testing.assert_array_almost_equal(ts_aps, ts_aps_m, decimal=4)
def test_resampled_tif_has_metadata(self):
small_test_ifgs = common.small_data_setup()
# minX, minY, maxX, maxY = extents
extents = [150.91, -34.229999976, 150.949166651, -34.17]
for s in small_test_ifgs:
resampled_temp_tif = tempfile.mktemp(suffix='.tif',
prefix='resampled_')
gdalwarp.resample_nearest_neighbour(s.data_path, extents,
[None, None],
resampled_temp_tif)
dst_ds = gdal.Open(resampled_temp_tif)
md = dst_ds.GetMetadata()
self.assertDictEqual(md, s.meta_data)
os.remove(resampled_temp_tif)
def test_small_data_cropping(self):
small_test_ifgs = common.small_data_setup()
# minX, minY, maxX, maxY = extents
extents = [150.91, -34.229999976, 150.949166651, -34.17]
extents_str = [str(e) for e in extents]
cmd = ['gdalwarp', '-overwrite', '-srcnodata', 'None', '-q', '-te'] \
+ extents_str
for s in small_test_ifgs:
temp_tif = tempfile.mktemp(suffix='.tif')
t_cmd = cmd + [s.data_path, temp_tif]
subprocess.check_call(t_cmd)
clipped_ref = gdal.Open(temp_tif).ReadAsArray()
clipped = gdalwarp.crop(s.data_path, extents)[0]
np.testing.assert_array_almost_equal(clipped_ref, clipped)
os.remove(temp_tif)
def test_slpfilter_auto_cutoff(slpfilter_method=2):
ts_aps_before = copy.copy(ts_hp_before_slpfilter)
params[cf.SLPF_METHOD] = slpfilter_method
params[cf.SLPF_CUTOFF] = 0
ifgs = small_data_setup()
Ifg = namedtuple('Ifg', 'x_centre, y_centre, x_size, y_size, shape')
ifg = Ifg(x_size=xpsize,
y_size=ypsize,
shape=ifgs[0].shape,
x_centre=ifgs[0].x_centre,
y_centre=ifgs[0].y_centre)
ts_aps = spatial_low_pass_filter(ts_aps_before, ifg, params=params)
for i in ifgs:
i.close()
np.testing.assert_array_almost_equal(ts_aps, ts_aps_m_auto, decimal=4)
def test_resampled_tif_has_metadata(self):
small_test_ifgs = common.small_data_setup()
# minX, minY, maxX, maxY = extents
extents = [150.91, -34.229999976, 150.949166651, -34.17]
for s in small_test_ifgs:
resampled_temp_tif = tempfile.mktemp(suffix='.tif',
prefix='resampled_')
gdal_python.resample_nearest_neighbour(
s.data_path, extents, [None, None], resampled_temp_tif)
dst_ds = gdal.Open(resampled_temp_tif)
md = dst_ds.GetMetadata()
self.assertDictEqual(md, s.meta_data)
try:
os.remove(resampled_temp_tif)
except PermissionError:
print("File opened by another process.")
def setUp(self):
from tests.common import small_data_setup
self.ifgs = small_data_setup()
self.ifg_paths = [i.data_path for i in self.ifgs]
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
self.headers = [
roipac.roipac_header(i.data_path, params) for i in self.ifgs
]
prepare_ifgs(self.ifg_paths,
crop_opt=1,
xlooks=1,
ylooks=1,
headers=self.headers)
looks_paths = [
mlooked_path(d, looks=1, crop_out=1) for d in self.ifg_paths
]
self.ifgs_with_nan = [Ifg(i) for i in looks_paths]
for ifg in self.ifgs_with_nan:
ifg.open()
def test_slpfilter_matlab(slpfilter_method):
# TODO: write tests for alpha = 0 special case when alpha (1/cutoff) is
# computed
params[cf.SLPF_METHOD] = slpfilter_method
ifgs = small_data_setup()
ts_aps_m = ts_aps_m1 if slpfilter_method == 1 else ts_aps_m2
ts_aps = np.zeros_like(ts_aps_m)
rows, cols = ifgs[0].shape
for i in range(ts_aps.shape[2]):
ts_aps[:, :, i] = _slp_filter(cutoff=params[cf.SLPF_CUTOFF],
rows=rows,
cols=cols,
x_size=xpsize,
y_size=ypsize,
params=params,
phase=ts_hp_before_slpfilter[:, :, i])
for i in ifgs:
i.close()
np.testing.assert_array_almost_equal(ts_aps, ts_aps_m, decimal=4)
def parallel_ifgs(gamma_conf):
tdir = Path(tempfile.mkdtemp())
params_p = manipulate_test_conf(gamma_conf, tdir)
params_p[cf.PARALLEL] = 1
output_conf_file = 'conf.conf'
output_conf = tdir.joinpath(output_conf_file)
cf.write_config_file(params=params_p, output_conf_file=output_conf)
params_p = Configuration(output_conf).__dict__
gtif_paths = conv2tif.main(params_p)
prepifg.main(params_p)
parallel_df = list(Path(tdir).joinpath('out').glob(glob_prefix))
p_ifgs = small_data_setup(datafiles=parallel_df)
yield p_ifgs
shutil.rmtree(params_p[cf.OBS_DIR])
def setup_class(cls):
from tests.common import small_data_setup
cls.ifgs = small_data_setup()
cls.ifg_paths = [i.data_path for i in cls.ifgs]
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
cls.headers = [
roipac.roipac_header(i.data_path, params) for i in cls.ifgs
]
params[C.IFG_LKSX], params[C.IFG_LKSY], params[
C.IFG_CROP_OPT] = 1, 1, 1
prepare_ifgs(cls.ifg_paths,
crop_opt=1,
xlooks=1,
ylooks=1,
headers=cls.headers,
params=params)
looks_paths = [
mlooked_path(d, params, t)
for d, t in zip(cls.ifg_paths, [InputTypes.IFG] * len(cls.ifgs))
]
cls.ifgs_with_nan = [Ifg(i) for i in looks_paths]
for ifg in cls.ifgs_with_nan:
ifg.open()
def test_gdal_python_vs_old_prepifg(self):
self.ifgs = common.small_data_setup(
datafiles=glob.glob(os.path.join(self.test_dir, "*.tif")))
for ifg in self.ifgs:
extents = [150.91, -34.229999976, 150.949166651, -34.17]
extents_str = [str(e) for e in extents]
orig_res = 0.000833333
thresh = 0.5
for looks in range(1, 10):
x_looks = y_looks = looks
res = orig_res*x_looks
averaged_and_resampled = gdal_python.crop_resample_average(
ifg.data_path, extents, new_res=[res, -res],
output_file=self.temp_tif, thresh=thresh,
match_pyrate=True)[0]
# only band 1 is resampled in warp_old
data, self.old_prepifg_path = warp_old(
ifg, x_looks, y_looks, extents_str, [res, -res],
thresh=thresh, crop_out=4, verbose=False)
yres, xres = data.shape
# old_prepifg warp resample method loses one row
# at the bottom if nrows % 2 == 1
np.testing.assert_array_almost_equal(data, averaged_and_resampled[:yres, :xres], decimal=4)
nrows, ncols = averaged_and_resampled.shape
# make sure they are the same after they are opened again
data_from_file = gdal.Open(self.old_prepifg_path).ReadAsArray()
self.assertTrue(os.path.exists(self.temp_tif))
new_from_file = gdal.Open(self.temp_tif).ReadAsArray()
np.testing.assert_array_almost_equal(data_from_file, new_from_file)
del averaged_and_resampled
del new_from_file
del data_from_file
def parallel_ifgs(gamma_conf):
tdir = Path(tempfile.mkdtemp())
params_p = manipulate_test_conf(gamma_conf, tdir)
params_p[C.PARALLEL] = 1
output_conf_file = 'conf.conf'
output_conf = tdir.joinpath(output_conf_file)
pyrate.configuration.write_config_file(params=params_p,
output_conf_file=output_conf)
params_p = Configuration(output_conf).__dict__
conv2tif.main(params_p)
prepifg.main(params_p)
parallel_df = list(
Path(params_p[C.INTERFEROGRAM_DIR]).glob(ifg_glob_suffix))
parallel_coh_files = list(
Path(params_p[C.COHERENCE_DIR]).glob(coh_glob_suffix))
p_ifgs = small_data_setup(datafiles=parallel_df + parallel_coh_files)
yield p_ifgs
def setUpClass(cls):
cls.tif_dir = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_GAMMA
# change the required params
cls.params = Configuration(cls.test_conf).__dict__
cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA
cls.params[cf.PROCESSOR] = 1 # gamma
file_list = list(
cf.parse_namelist(os.path.join(common.SML_TEST_GAMMA, 'ifms_17')))
fd, cls.params[cf.IFG_FILE_LIST] = tempfile.mkstemp(suffix='.conf',
dir=cls.tif_dir)
os.close(fd)
# write a short filelist with only 3 gamma unws
with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp:
for f in file_list[:3]:
fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n')
cls.params[cf.OUT_DIR] = cls.tif_dir
cls.params[cf.PARALLEL] = 0
cls.params[cf.REF_EST_METHOD] = 1
cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
# base_unw_paths need to be geotiffed and multilooked by run_prepifg
cls.base_unw_paths = cf.original_ifg_paths(
cls.params[cf.IFG_FILE_LIST], cls.params[cf.OBS_DIR])
cls.base_unw_paths.append(common.SML_TEST_DEM_GAMMA)
xlks, ylks, crop = cf.transform_params(cls.params)
# dest_paths are tifs that have been geotif converted and multilooked
conv2tif.main(cls.params)
prepifg.main(cls.params)
# run_prepifg.gamma_prepifg(cls.base_unw_paths, cls.params)
cls.base_unw_paths.pop() # removed dem as we don't want it in ifgs
cls.dest_paths = cf.get_dest_paths(cls.base_unw_paths, crop,
cls.params, xlks)
cls.ifgs = common.small_data_setup(datafiles=cls.dest_paths)
def setup_class(cls, roipac_params):
cls.params = roipac_params
cls.ifgs = common.small_data_setup()
# read in input (tscuml) and expected output arrays
tscuml_path = os.path.join(common.SML_TEST_LINRATE, "tscuml_0.npy")
cls.tscuml0 = np.load(tscuml_path)
# add zero epoch to tscuml 3D array
cls.tscuml = np.insert(cls.tscuml0, 0, 0, axis=2)
linrate_path = os.path.join(common.SML_TEST_LINRATE, "linear_rate.npy")
cls.linrate = np.load(linrate_path)
error_path = os.path.join(common.SML_TEST_LINRATE, "linear_error.npy")
cls.error = np.load(error_path)
icpt_path = os.path.join(common.SML_TEST_LINRATE, "linear_intercept.npy")
cls.icpt = np.load(icpt_path)
samp_path = os.path.join(common.SML_TEST_LINRATE, "linear_samples.npy")
cls.samp = np.load(samp_path)
rsq_path = os.path.join(common.SML_TEST_LINRATE, "linear_rsquared.npy")
cls.rsq = np.load(rsq_path)
def parallel_ifgs(gamma_conf):
tdir = Path(tempfile.mkdtemp())
params_p = manipulate_test_conf(gamma_conf, tdir)
params_p[cf.PARALLEL] = 1
output_conf_file = 'conf.conf'
output_conf = tdir.joinpath(output_conf_file)
pyrate.configuration.write_config_file(params=params_p,
output_conf_file=output_conf)
params_p = Configuration(output_conf).__dict__
conv2tif.main(params_p)
prepifg.main(params_p)
parallel_df = list(Path(tdir).joinpath('out').glob(ifg_glob_suffix))
parallel_coh_files = list(Path(tdir).joinpath('out').glob(coh_glob_suffix))
p_ifgs = small_data_setup(datafiles=parallel_df + parallel_coh_files)
yield p_ifgs
shutil.rmtree(params_p[cf.OBS_DIR], ignore_errors=True)
def setUpClass(cls):
cls.tif_dir = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_GAMMA
# change the required params
cls.params = Configuration(cls.test_conf).__dict__
cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA
cls.params[cf.PROCESSOR] = 1 # gamma
file_list = list(
cf.parse_namelist(os.path.join(common.SML_TEST_GAMMA, 'ifms_17')))
fd, cls.params[cf.IFG_FILE_LIST] = tempfile.mkstemp(suffix='.conf',
dir=cls.tif_dir)
os.close(fd)
# write a short filelist with only 3 gamma unws
with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp:
for f in file_list[:3]:
fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n')
cls.params[cf.OUT_DIR] = cls.tif_dir
cls.params[cf.PARALLEL] = 0
cls.params[cf.REF_EST_METHOD] = 1
cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
# base_unw_paths need to be geotiffed and multilooked by run_prepifg
cls.base_unw_paths = cf.original_ifg_paths(
cls.params[cf.IFG_FILE_LIST], cls.params[cf.OBS_DIR])
cls.base_unw_paths.append(common.SML_TEST_DEM_GAMMA)
# dest_paths are tifs that have been geotif converted and multilooked
conv2tif.main(cls.params)
prepifg.main(cls.params)
cls.dest_paths = [
Path(cls.tif_dir).joinpath(Path(c.sampled_path).name).as_posix()
for c in cls.params[cf.INTERFEROGRAM_FILES][:-2]
]
cls.ifgs = common.small_data_setup(datafiles=cls.dest_paths)
def setup_method(self):
self.ifgs = small_data_setup()
def setup_class(cls):
gamma_conf = common.TEST_CONF_GAMMA
from tests.common import manipulate_test_conf
rate_types = ['stack_rate', 'stack_error', 'stack_samples']
cls.tif_dir = Path(tempfile.mkdtemp())
params = manipulate_test_conf(gamma_conf, cls.tif_dir)
from pyrate.configuration import Configuration
# change the required params
params[C.PROCESSES] = 4
params[C.PROCESSOR] = 1 # gamma
params[C.IFG_FILE_LIST] = os.path.join(common.GAMMA_SML_TEST_DIR,
'ifms_17')
params[C.PARALLEL] = 1
params[C.APS_CORRECTION] = 0
params[C.REFX], params[C.REFY] = -1, -1
rows, cols = params["rows"], params["cols"]
output_conf_file = 'gamma.conf'
output_conf = cls.tif_dir.joinpath(output_conf_file).as_posix()
pyrate.configuration.write_config_file(params=params,
output_conf_file=output_conf)
config = Configuration(output_conf)
params = config.__dict__
common.sub_process_run(f"pyrate conv2tif -f {output_conf}")
common.sub_process_run(f"pyrate prepifg -f {output_conf}")
cls.sampled_paths = [
p.tmp_sampled_path for p in params[C.INTERFEROGRAM_FILES]
]
ifgs = common.small_data_setup()
correct._copy_mlooked(params)
tiles = pyrate.core.shared.get_tiles(cls.sampled_paths[0], rows, cols)
correct.correct_ifgs(config)
pyrate.main.timeseries(config)
pyrate.main.stack(config)
cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = \
(params[C.REFX], params[C.REFY]), params[C.MAXVAR], params[
C.VCMT]
cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles,
params[C.OUT_DIR])
cls.rate_p, cls.error_p, cls.samples_p = \
[common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[C.TMPDIR], t) for t in rate_types]
common.remove_tifs(params[C.OUT_DIR])
# now create the non parallel version
cls.tif_dir_s = Path(tempfile.mkdtemp())
params = manipulate_test_conf(gamma_conf, cls.tif_dir_s)
params[C.PROCESSES] = 4
params[C.PROCESSOR] = 1 # gamma
params[C.IFG_FILE_LIST] = os.path.join(common.GAMMA_SML_TEST_DIR,
'ifms_17')
params[C.PARALLEL] = 0
params[C.APS_CORRECTION] = 0
params[C.REFX], params[C.REFY] = -1, -1
output_conf_file = 'gamma.conf'
output_conf = cls.tif_dir_s.joinpath(output_conf_file).as_posix()
pyrate.configuration.write_config_file(params=params,
output_conf_file=output_conf)
config = Configuration(output_conf)
params = config.__dict__
common.sub_process_run(f"pyrate conv2tif -f {output_conf}")
common.sub_process_run(f"pyrate prepifg -f {output_conf}")
correct._copy_mlooked(params)
correct.correct_ifgs(config)
pyrate.main.timeseries(config)
pyrate.main.stack(config)
cls.refpixel, cls.maxvar, cls.vcmt = \
(params[C.REFX], params[C.REFY]), params[C.MAXVAR], params[
C.VCMT]
cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles,
params[C.OUT_DIR])
cls.rate, cls.error, cls.samples = \
[common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[C.TMPDIR], t) for t in rate_types]
cls.params = params
def setup_class(cls):
cls.ifgs = small_data_setup()
def setUpClass(cls):
rate_types = ['stack_rate', 'stack_error', 'stack_samples']
cls.tif_dir = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_GAMMA
from pyrate.configuration import Configuration
# change the required params
params = Configuration(cls.test_conf).__dict__
params[cf.OBS_DIR] = common.SML_TEST_GAMMA
params[cf.PROCESSES] = 4
params[cf.PROCESSOR] = 1 # gamma
params[cf.IFG_FILE_LIST] = os.path.join(common.SML_TEST_GAMMA,
'ifms_17')
params[cf.OUT_DIR] = cls.tif_dir
params[cf.PARALLEL] = 1
params[cf.APS_CORRECTION] = False
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
rows, cols = params["rows"], params["cols"]
# xlks, ylks, crop = cf.transform_params(params)
# base_unw_paths need to be geotiffed by converttogeotif
# and multilooked by run_prepifg
base_unw_paths = list(cf.parse_namelist(params[cf.IFG_FILE_LIST]))
multi_paths = [
MultiplePaths(params[cf.OUT_DIR],
b,
ifglksx=params[cf.IFG_LKSX],
ifgcropopt=params[cf.IFG_CROP_OPT])
for b in base_unw_paths
]
# dest_paths are tifs that have been geotif converted and multilooked
cls.converted_paths = [b.converted_path for b in multi_paths]
cls.sampled_paths = [b.sampled_path for b in multi_paths]
from copy import copy
orig_params = copy(params)
conv2tif.main(params)
prepifg.main(orig_params)
tiles = pyrate.core.shared.get_tiles(cls.sampled_paths[0], rows, cols)
ifgs = common.small_data_setup()
params[cf.INTERFEROGRAM_FILES] = multi_paths
cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = process.process_ifgs(
cls.sampled_paths, params, rows, cols)
cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate_p, cls.error_p, cls.samples_p = \
[common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types]
common.remove_tifs(params[cf.OBS_DIR])
# now create the non parallel version
cls.tif_dir_s = tempfile.mkdtemp()
params[cf.PARALLEL] = 0
params[cf.PROCESSES] = 1
params[cf.OUT_DIR] = cls.tif_dir_s
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
multi_paths = [
MultiplePaths(params[cf.OUT_DIR],
b,
ifglksx=params[cf.IFG_LKSX],
ifgcropopt=params[cf.IFG_CROP_OPT])
for b in base_unw_paths
]
cls.converted_paths_s = [b.converted_path for b in multi_paths]
cls.sampled_paths_s = [b.sampled_path for b in multi_paths]
orig_params = copy(params)
conv2tif.main(params)
prepifg.main(orig_params)
params[cf.INTERFEROGRAM_FILES] = multi_paths
cls.refpixel, cls.maxvar, cls.vcmt = process.process_ifgs(
cls.sampled_paths_s, params, rows, cols)
cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate, cls.error, cls.samples = \
[common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types]
def test_timeseries_linrate_mpi(mpisync, tempdir, modify_config,
ref_est_method, row_splits, col_splits,
get_crop, orbfit_lks, orbfit_method,
orbfit_degrees):
params = modify_config
outdir = mpiops.run_once(tempdir)
params[cf.OUT_DIR] = outdir
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
params[cf.DEM_HEADER_FILE] = SML_TEST_DEM_HDR_GAMMA
params[cf.REF_EST_METHOD] = ref_est_method
params[cf.IFG_CROP_OPT] = get_crop
params[cf.ORBITAL_FIT_LOOKS_Y] = orbfit_lks
params[cf.ORBITAL_FIT_LOOKS_X] = orbfit_lks
params[cf.ORBITAL_FIT_METHOD] = orbfit_method
params[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees
xlks, ylks, crop = cf.transform_params(params)
if xlks * col_splits > 45 or ylks * row_splits > 70:
print('skipping test because lks and col_splits are not compatible')
return
# skip some tests in travis to run CI faster
if TRAVIS and (xlks % 2 or row_splits % 2 or col_splits % 2
or orbfit_lks % 2):
print('Skipping in travis env for faster CI run')
return
print("xlks={}, ref_est_method={}, row_splits={}, col_splits={}, "
"get_crop={}, orbfit_lks={}, orbfit_method={}, "
"rank={}".format(xlks, ref_est_method, row_splits, col_splits,
get_crop, orbfit_lks, orbfit_method, orbfit_degrees,
mpiops.rank))
base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
# dest_paths are tifs that have been geotif converted and multilooked
dest_paths = cf.get_dest_paths(base_unw_paths, crop, params, xlks)
# run prepifg, create the dest_paths files
if mpiops.rank == 0:
run_prepifg.gamma_prepifg(base_unw_paths, params)
mpiops.comm.barrier()
(refpx,
refpy), maxvar, vcmt = run_pyrate.process_ifgs(ifg_paths=dest_paths,
params=params,
rows=row_splits,
cols=col_splits)
tiles = mpiops.run_once(run_pyrate.get_tiles,
dest_paths[0],
rows=row_splits,
cols=col_splits)
postprocessing.postprocess_linrate(row_splits, col_splits, params)
postprocessing.postprocess_timeseries(row_splits, col_splits, params)
ifgs_mpi_out_dir = params[cf.OUT_DIR]
ifgs_mpi = small_data_setup(datafiles=dest_paths)
# single process timeseries/linrate calculation
if mpiops.rank == 0:
params_old = modify_config
params_old[cf.OUT_DIR] = tempdir()
params_old[cf.REF_EST_METHOD] = ref_est_method
params_old[cf.IFG_CROP_OPT] = get_crop
params_old[cf.ORBITAL_FIT_LOOKS_Y] = orbfit_lks
params_old[cf.ORBITAL_FIT_LOOKS_X] = orbfit_lks
params_old[cf.ORBITAL_FIT_METHOD] = orbfit_method
params_old[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees
xlks, ylks, crop = cf.transform_params(params_old)
base_unw_paths = cf.original_ifg_paths(params_old[cf.IFG_FILE_LIST])
dest_paths = cf.get_dest_paths(base_unw_paths, crop, params_old, xlks)
run_prepifg.gamma_prepifg(base_unw_paths, params_old)
ifgs = shared.pre_prepare_ifgs(dest_paths, params_old)
mst_grid = tests.common.mst_calculation(dest_paths, params_old)
refy, refx = refpixel.ref_pixel(ifgs, params_old)
assert (refx == refpx) and (refy == refpy) # both must match
pyrate.orbital.remove_orbital_error(ifgs, params_old)
ifgs = common.prepare_ifgs_without_phase(dest_paths, params_old)
rpe.estimate_ref_phase(ifgs, params_old, refx, refy)
ifgs = shared.pre_prepare_ifgs(dest_paths, params_old)
maxvar_s = [vcm.cvd(i, params_old)[0] for i in ifgs]
vcmt_s = vcm.get_vcmt(ifgs, maxvar)
tsincr, tscum, _ = tests.common.compute_time_series(
ifgs, mst_grid, params, vcmt)
rate, error, samples = tests.common.calculate_linear_rate(
ifgs, params_old, vcmt, mst_grid)
mst_mpi = reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR])
np.testing.assert_array_almost_equal(mst_grid, mst_mpi)
tsincr_mpi, tscum_mpi = reconstruct_times_series(
ifgs[0].shape, tiles, params[cf.TMPDIR])
rate_mpi, error_mpi, samples_mpi = \
[reconstruct_linrate(ifgs[0].shape, tiles, params[cf.TMPDIR], t)
for t in ['linrate', 'linerror', 'linsamples']]
np.testing.assert_array_almost_equal(maxvar, maxvar_s)
np.testing.assert_array_almost_equal(vcmt, vcmt_s)
for i, j in zip(ifgs, ifgs_mpi):
np.testing.assert_array_almost_equal(i.phase_data, j.phase_data)
np.testing.assert_array_almost_equal(tsincr, tsincr_mpi, decimal=4)
np.testing.assert_array_almost_equal(tscum, tscum_mpi, decimal=4)
np.testing.assert_array_almost_equal(rate, rate_mpi, decimal=4)
np.testing.assert_array_almost_equal(error, error_mpi, decimal=4)
np.testing.assert_array_almost_equal(samples, samples_mpi, decimal=4)
# assert linear rate output tifs are same
_tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'linrate.tif')
_tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'linerror.tif')
_tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'linsamples.tif')
# assert time series output tifs are same
epochlist = algorithm.get_epochs(ifgs)[0]
for i in range(tsincr.shape[2]):
_tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR],
'tsincr' + '_' + str(epochlist.dates[i + 1]) + ".tif")
# 12 timeseries outputs
assert i + 1 == tsincr.shape[2]
shutil.rmtree(ifgs_mpi_out_dir) # remove mpi out dir
shutil.rmtree(params_old[cf.OUT_DIR]) # remove serial out dir
def setUpClass(cls):
cls.tif_dir_inc = tempfile.mkdtemp()
cls.tif_dir_ele = tempfile.mkdtemp()
cls.tif_dir_ele_par = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_ROIPAC
# change the required params
cls.params_inc = cf.get_config_params(cls.test_conf)
cls.params_inc[cf.OBS_DIR] = common.SML_TEST_GAMMA
cls.params_inc[cf.PROCESSOR] = 1 # gamma
file_list = cf.parse_namelist(
os.path.join(common.SML_TEST_GAMMA, 'ifms_17'))
# config file
cls.params_inc[cf.IFG_FILE_LIST] = tempfile.mktemp(dir=cls.tif_dir_inc)
# write a short filelist with only 3 gamma unws
with open(cls.params_inc[cf.IFG_FILE_LIST], 'w') as fp:
for f in file_list[:2]:
fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n')
cls.params_inc[cf.OUT_DIR] = cls.tif_dir_inc
cls.params_inc[cf.PARALLEL] = 0
cls.params_inc[cf.REF_EST_METHOD] = 1
cls.params_inc[cf.APS_METHOD] = 2
cls.params_inc[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
cls.params_inc[cf.APS_INCIDENCE_MAP] = common.SML_TEST_INCIDENCE
run_prepifg.main(cls.params_inc)
# now create the config for the elevation_map case
cls.params_ele = copy.copy(cls.params_inc)
cls.params_ele[cf.OUT_DIR] = cls.tif_dir_ele
cls.params_ele[cf.APS_METHOD] = 2
cls.params_ele[cf.APS_INCIDENCE_MAP] = None
cls.params_ele[cf.APS_INCIDENCE_EXT] = None
cls.params_ele[cf.APS_ELEVATION_MAP] = common.SML_TEST_ELEVATION
cls.params_ele[cf.APS_ELEVATION_EXT] = 'lv_theta'
run_prepifg.main(cls.params_ele)
ptn = re.compile(r'\d{8}')
dest_paths_inc = [
f for f in glob.glob(os.path.join(cls.tif_dir_inc, '*.tif'))
if ("cr" in f) and ("rlks" in f) and (
len(re.findall(ptn, os.path.basename(f))) == 2)
]
cls.ifgs_inc = common.small_data_setup(datafiles=dest_paths_inc)
dest_paths_ele = [
f for f in glob.glob(os.path.join(cls.tif_dir_ele, '*.tif'))
if "cr" in f and "rlks" in f and (
len(re.findall(ptn, os.path.basename(f))) == 2)
]
cls.ifgs_ele = common.small_data_setup(datafiles=dest_paths_ele)
# now create the config for the elevation map parallel case
cls.params_ele_par = copy.copy(cls.params_ele)
cls.params_ele_par[cf.OUT_DIR] = cls.tif_dir_ele_par
cls.params_ele_par[cf.PARALLEL] = True
run_prepifg.main(cls.params_ele_par)
dest_paths_ele_par = \
[f for f in glob.glob(os.path.join(cls.tif_dir_ele_par, '*.tif'))
if "cr" in f and "rlks" in f and
(len(re.findall(ptn, os.path.basename(f))) == 2)]
cls.ifgs_ele_par = common.small_data_setup(
datafiles=dest_paths_ele_par)
aps.remove_aps_delay(cls.ifgs_inc, cls.params_inc)
aps.remove_aps_delay(cls.ifgs_ele, cls.params_ele)
aps.remove_aps_delay(cls.ifgs_ele_par, cls.params_ele_par)
def test_orbital_correction(self):
key = 'ORBITAL_ERROR'
value = 'REMOVED'
for i in common.small_data_setup(datafiles=self.dest_paths):
self.key_check(i, key, value)
def setUpClass(cls):
rate_types = ['linrate', 'linerror', 'linsamples']
cls.tif_dir = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_GAMMA
# change the required params
params = cf.get_config_params(cls.test_conf)
params[cf.OBS_DIR] = common.SML_TEST_GAMMA
params[cf.PROCESSOR] = 1 # gamma
params[cf.IFG_FILE_LIST] = os.path.join(
common.SML_TEST_GAMMA, 'ifms_17')
params[cf.OUT_DIR] = cls.tif_dir
params[cf.PARALLEL] = 1
params[cf.APS_CORRECTION] = False
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
xlks, ylks, crop = cf.transform_params(params)
# base_unw_paths need to be geotiffed by converttogeotif
# and multilooked by run_prepifg
base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST],
params[cf.OBS_DIR])
# dest_paths are tifs that have been geotif converted and multilooked
cls.dest_paths = cf.get_dest_paths(
base_unw_paths, crop, params, xlks)
gtif_paths = conv2tif.do_geotiff(base_unw_paths, params)
prepifg.do_prepifg(gtif_paths, params)
tiles = pyrate.core.shared.get_tiles(cls.dest_paths[0], 3, 3)
ifgs = common.small_data_setup()
cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = \
process.process_ifgs(cls.dest_paths, params, 3, 3)
cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate_p, cls.error_p, cls.samples_p = [
common.reconstruct_linrate(
ifgs[0].shape, tiles, params[cf.TMPDIR], t)
for t in rate_types
]
common.remove_tifs(params[cf.OBS_DIR])
# now create the non parallel version
cls.tif_dir_s = tempfile.mkdtemp()
params[cf.PARALLEL] = 0
params[cf.OUT_DIR] = cls.tif_dir_s
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
cls.dest_paths_s = cf.get_dest_paths(
base_unw_paths, crop, params, xlks)
gtif_paths = conv2tif.do_geotiff(base_unw_paths, params)
prepifg.do_prepifg(gtif_paths, params)
cls.refpixel, cls.maxvar, cls.vcmt = \
process.process_ifgs(cls.dest_paths_s, params, 3, 3)
cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate, cls.error, cls.samples = [
common.reconstruct_linrate(
ifgs[0].shape, tiles, params[cf.TMPDIR], t)
for t in rate_types
]
def setUp(self):
self.ifgs = small_data_setup()
def setUp(self):
self.ifgs = small_data_setup()
self.params = cf.get_config_params(common.TEST_CONF_ROIPAC)
def setUpClass(cls):
cls.tif_dir_serial = tempfile.mkdtemp()
cls.tif_dir_mpi = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_ROIPAC
# change the required params
cls.params = cf.get_config_params(cls.test_conf)
cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA
cls.params[cf.PROCESSOR] = 1 # gamma
file_list = cf.parse_namelist(
os.path.join(common.SML_TEST_GAMMA, 'ifms_17'))
cls.params[cf.IFG_FILE_LIST] = tempfile.mktemp(dir=cls.tif_dir_serial)
# write a short filelist with only 3 gamma unws
with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp:
for f in file_list[:2]:
fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n')
cls.params[cf.OUT_DIR] = cls.tif_dir_serial
cls.params[cf.PARALLEL] = 0
cls.params[cf.REF_EST_METHOD] = 2
cls.params[cf.IFG_LKSX] = 1
cls.params[cf.IFG_LKSY] = 1
cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
cls.params[cf.APS_INCIDENCE_MAP] = common.SML_TEST_INCIDENCE
# base_unw_paths need to be geotiffed and multilooked by run_prepifg
base_unw_paths = run_pyrate.original_ifg_paths(
cls.params[cf.IFG_FILE_LIST])
# add dem
base_unw_paths.append(common.SML_TEST_DEM_GAMMA)
# add incidence
base_unw_paths.append(common.SML_TEST_INCIDENCE)
xlks, ylks, crop = run_pyrate.transform_params(cls.params)
cls.params_mpi = copy.copy(cls.params)
cls.params_mpi[cf.OUT_DIR] = cls.tif_dir_mpi
# dest_paths are tifs that have been geotif converted and multilooked
run_prepifg.gamma_prepifg(base_unw_paths, cls.params)
run_prepifg.gamma_prepifg(base_unw_paths, cls.params_mpi)
# removed incidence as we don't want it in ifgs list
base_unw_paths.pop()
# removed dem as we don't want it in ifgs list
base_unw_paths.pop()
dest_paths = run_pyrate.get_dest_paths(base_unw_paths, crop,
cls.params, xlks)
dest_paths_mpi = run_pyrate.get_dest_paths(base_unw_paths, crop,
cls.params_mpi, xlks)
run_pyrate.process_ifgs(dest_paths, cls.params)
cls.ifgs_serial = common.small_data_setup(datafiles=dest_paths)
cls.conf_mpi = tempfile.mktemp('.conf', dir=cls.tif_dir_mpi)
cf.write_config_file(cls.params_mpi, cls.conf_mpi)
str = 'mpirun -np 4 python pyrate/nci/run_pyrate_pypar.py ' + \
cls.conf_mpi
cmd = str.split()
subprocess.check_call(cmd)
str = 'mpirun -np 4 python pyrate/nci/run_pyrate_pypar_2.py ' + \
cls.conf_mpi
cmd = str.split()
subprocess.check_call(cmd)
str = 'mpirun -np 4 python pyrate/nci/run_pyrate_pypar_3.py ' + \
cls.conf_mpi
cmd = str.split()
subprocess.check_call(cmd)
cls.ifgs_mpi = common.small_data_setup(datafiles=dest_paths_mpi)
def setUpClass(cls):
cls.ifgs = common.small_data_setup()
cls.params = default_params()
cls.mstmat = mst.mst_boolean_array(cls.ifgs)
cls.maxvar = [vcm.cvd(i, cls.params)[0] for i in cls.ifgs]
cls.vcmt = vcm.get_vcmt(cls.ifgs, cls.maxvar)