def setUpClass(cls):
# read in the params
_, _, params = cf.get_ifg_paths(TEST_CONF_GAMMA)
glob_prefix = "*utm_unw_1rlks_1cr.tif"
# SERIAL
cls.serial_dir = tempfile.mkdtemp()
params[cf.OUT_DIR] = cls.serial_dir
params[cf.PARALLEL] = False
shared.mkdir_p(cls.serial_dir)
gtif_paths = conv2tif.main(params)
prepifg.main(params)
serial_df = glob.glob(os.path.join(cls.serial_dir, glob_prefix))
cls.serial_ifgs = small_data_setup(datafiles=serial_df)
# Clean up serial converted tifs so we can test parallel conversion
common.remove_tifs(SML_TEST_GAMMA)
# PARALLEL
cls.parallel_dir = tempfile.mkdtemp()
params[cf.OUT_DIR] = cls.parallel_dir
params[cf.PARALLEL] = True
shared.mkdir_p(cls.parallel_dir)
gtif_paths = conv2tif.main(params)
prepifg.main(params)
para_df = glob.glob(os.path.join(cls.parallel_dir, glob_prefix))
cls.para_ifgs = small_data_setup(datafiles=para_df)
def test_prepifg_mpi(mpisync, get_config, tempdir, roipac_or_gamma, get_lks,
get_crop):
from tests.common import TEST_CONF_ROIPAC, TEST_CONF_GAMMA
from os.path import join, basename
if roipac_or_gamma == 1:
params = get_config(TEST_CONF_GAMMA)
else:
params = get_config(TEST_CONF_ROIPAC)
outdir = mpiops.run_once(tempdir)
params[cf.OUT_DIR] = outdir
params[cf.PARALLEL] = False
params[cf.IFG_LKSX], params[cf.IFG_LKSY] = get_lks, get_lks
params[cf.IFG_CROP_OPT] = get_crop
if roipac_or_gamma == 1:
params[cf.IFG_FILE_LIST] = join(common.SML_TEST_GAMMA, 'ifms_17')
params[cf.OBS_DIR] = common.SML_TEST_GAMMA
params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
params[cf.DEM_HEADER_FILE] = common.SML_TEST_DEM_HDR_GAMMA
conv2tif.main(params)
prepifg.main(params)
common.remove_tifs(params[cf.OBS_DIR])
if mpiops.rank == 0:
if roipac_or_gamma == 1:
params_s = get_config(TEST_CONF_GAMMA)
else:
params_s = get_config(TEST_CONF_ROIPAC)
params_s[cf.OUT_DIR] = tempdir()
params_s[cf.PARALLEL] = True
params_s[cf.IFG_LKSX], params_s[cf.IFG_LKSY] = get_lks, get_lks
params_s[cf.IFG_CROP_OPT] = get_crop
conv2tif.main(params)
if roipac_or_gamma == 1:
base_unw_paths = glob.glob(join(common.SML_TEST_GAMMA,
"*_utm.unw"))
prepifg.main(params)
else:
base_unw_paths = glob.glob(join(common.SML_TEST_OBS, "*.unw"))
prepifg.main(params_s)
mpi_tifs = glob.glob(join(outdir, "*.tif"))
serial_tifs = glob.glob(join(params[cf.OUT_DIR], "*.tif"))
mpi_tifs.sort()
serial_tifs.sort()
# 17 geotifs, and 17 mlooked tifs
assert len(mpi_tifs) == len(serial_tifs)
for m_f, s_f in zip(mpi_tifs, serial_tifs):
assert basename(m_f) == basename(s_f)
shutil.rmtree(outdir)
shutil.rmtree(params_s[cf.OUT_DIR])
common.remove_tifs(params[cf.OBS_DIR])
def tearDownClass(cls):
try:
shutil.rmtree(cls.temp_out_dir)
except PermissionError:
print("File opened by another process.")
try:
common.remove_tifs(
cf.get_config_params(common.TEST_CONF_ROIPAC)[cf.OBS_DIR])
except PermissionError:
print("File opened by another process.")
for ifg in cls.ifgs:
ifg.close()
def tearDownClass(cls):
try:
shutil.rmtree(cls.parallel_dir)
except PermissionError:
print("File opened by another process.")
try:
shutil.rmtree(cls.serial_dir)
except PermissionError:
print("File opened by another process.")
try:
common.remove_tifs(SML_TEST_GAMMA)
except PermissionError:
print("File opened by another process.")
def test_vcm_legacy_vs_mpi(mpisync, tempdir, get_config):
from tests.common import SML_TEST_DIR, TEST_CONF_ROIPAC
params_dict = get_config(TEST_CONF_ROIPAC)
LEGACY_VCM_DIR = os.path.join(SML_TEST_DIR, 'vcm')
legacy_vcm = np.genfromtxt(os.path.join(LEGACY_VCM_DIR, 'vcmt.csv'),
delimiter=',')
if mpiops.rank == 0:
outdir = tempdir()
else:
outdir = None
outdir = mpiops.comm.bcast(outdir, root=0)
params_dict[cf.OUT_DIR] = outdir
params_dict[cf.PARALLEL] = False
xlks, ylks, crop = cf.transform_params(params_dict)
base_unw_paths = cf.original_ifg_paths(params_dict[cf.IFG_FILE_LIST],
params_dict[cf.OBS_DIR])
# dest_paths are tifs that have been geotif converted and multilooked
dest_paths = cf.get_dest_paths(base_unw_paths, crop, params_dict, xlks)
# run prepifg, create the dest_paths files
if mpiops.rank == 0:
conv2tif.main(params_dict)
prepifg.main(params_dict)
mpiops.comm.barrier()
tiles = pyrate.core.shared.get_tiles(dest_paths[0], rows=1, cols=1)
preread_ifgs = process._create_ifg_dict(dest_paths,
params=params_dict,
tiles=tiles)
refpx, refpy = process._ref_pixel_calc(dest_paths, params_dict)
process._orb_fit_calc(dest_paths, params_dict)
process._ref_phase_estimation(dest_paths, params_dict, refpx, refpy)
maxvar, vcmt = process._maxvar_vcm_calc(dest_paths, params_dict,
preread_ifgs)
np.testing.assert_array_almost_equal(maxvar, legacy_maxvar, decimal=4)
np.testing.assert_array_almost_equal(legacy_vcm, vcmt, decimal=3)
if mpiops.rank == 0:
shutil.rmtree(outdir)
common.remove_tifs(params_dict[cf.OBS_DIR])
def tearDownClass(cls):
for ifg in cls.ifgs:
ifg.close()
shutil.rmtree(cls.temp_out_dir)
common.remove_tifs(
cf.get_config_params(common.TEST_CONF_ROIPAC)[cf.OBS_DIR])
def tearDownClass(cls):
for i in cls.ifgs:
i.close()
shutil.rmtree(cls.tif_dir)
common.remove_tifs(cls.params[cf.OBS_DIR])
def teardown_method(self, method):
common.remove_tifs(self.gamma_params[cf.OBS_DIR])
common.remove_tifs(self.roipac_params[cf.OBS_DIR])
def setUpClass(cls):
rate_types = ['linrate', 'linerror', 'linsamples']
cls.tif_dir = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_GAMMA
# change the required params
params = cf.get_config_params(cls.test_conf)
params[cf.OBS_DIR] = common.SML_TEST_GAMMA
params[cf.PROCESSOR] = 1 # gamma
params[cf.IFG_FILE_LIST] = os.path.join(
common.SML_TEST_GAMMA, 'ifms_17')
params[cf.OUT_DIR] = cls.tif_dir
params[cf.PARALLEL] = 1
params[cf.APS_CORRECTION] = False
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
xlks, ylks, crop = cf.transform_params(params)
# base_unw_paths need to be geotiffed by converttogeotif
# and multilooked by run_prepifg
base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST],
params[cf.OBS_DIR])
# dest_paths are tifs that have been geotif converted and multilooked
cls.dest_paths = cf.get_dest_paths(
base_unw_paths, crop, params, xlks)
gtif_paths = conv2tif.do_geotiff(base_unw_paths, params)
prepifg.do_prepifg(gtif_paths, params)
tiles = pyrate.core.shared.get_tiles(cls.dest_paths[0], 3, 3)
ifgs = common.small_data_setup()
cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = \
process.process_ifgs(cls.dest_paths, params, 3, 3)
cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate_p, cls.error_p, cls.samples_p = [
common.reconstruct_linrate(
ifgs[0].shape, tiles, params[cf.TMPDIR], t)
for t in rate_types
]
common.remove_tifs(params[cf.OBS_DIR])
# now create the non parallel version
cls.tif_dir_s = tempfile.mkdtemp()
params[cf.PARALLEL] = 0
params[cf.OUT_DIR] = cls.tif_dir_s
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
cls.dest_paths_s = cf.get_dest_paths(
base_unw_paths, crop, params, xlks)
gtif_paths = conv2tif.do_geotiff(base_unw_paths, params)
prepifg.do_prepifg(gtif_paths, params)
cls.refpixel, cls.maxvar, cls.vcmt = \
process.process_ifgs(cls.dest_paths_s, params, 3, 3)
cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate, cls.error, cls.samples = [
common.reconstruct_linrate(
ifgs[0].shape, tiles, params[cf.TMPDIR], t)
for t in rate_types
]
def tearDownClass(cls):
shutil.rmtree(cls.tif_dir, ignore_errors=True)
shutil.rmtree(cls.tif_dir_s, ignore_errors=True)
common.remove_tifs(cf.get_config_params(cls.test_conf)[cf.OBS_DIR])
def teardown_class(cls):
params = Configuration(cls.conf_file).__dict__
shutil.rmtree(cls.base_dir)
common.remove_tifs(params[WORKING_DIR])
def tearDownClass(cls):
shutil.rmtree(cls.gamma_base_dir)
shutil.rmtree(cls.roipac_base_dir)
remove_tifs(cls.SMLNEY_GAMMA_TEST)
remove_tifs(common.SML_TEST_OBS)
def teardown_method(self, method):
common.remove_tifs(self.params[cf.OBS_DIR])
def tearDown(self):
params = cf.get_config_params(self.conf_file)
shutil.rmtree(self.base_dir)
common.remove_tifs(params[cf.OBS_DIR])
def tearDownClass(cls):
shutil.rmtree(cls.temp_out_dir)
params = cf.get_config_params(TEST_CONF_ROIPAC)
remove_tifs(params[cf.OBS_DIR])
def setup_class(cls):
gamma_conf = common.TEST_CONF_GAMMA
from tests.common import manipulate_test_conf
rate_types = ['stack_rate', 'stack_error', 'stack_samples']
cls.tif_dir = Path(tempfile.mkdtemp())
params = manipulate_test_conf(gamma_conf, cls.tif_dir)
from pyrate.configuration import Configuration
# change the required params
params[C.PROCESSES] = 4
params[C.PROCESSOR] = 1 # gamma
params[C.IFG_FILE_LIST] = os.path.join(common.GAMMA_SML_TEST_DIR,
'ifms_17')
params[C.PARALLEL] = 1
params[C.APS_CORRECTION] = 0
params[C.REFX], params[C.REFY] = -1, -1
rows, cols = params["rows"], params["cols"]
output_conf_file = 'gamma.conf'
output_conf = cls.tif_dir.joinpath(output_conf_file).as_posix()
pyrate.configuration.write_config_file(params=params,
output_conf_file=output_conf)
config = Configuration(output_conf)
params = config.__dict__
common.sub_process_run(f"pyrate conv2tif -f {output_conf}")
common.sub_process_run(f"pyrate prepifg -f {output_conf}")
cls.sampled_paths = [
p.tmp_sampled_path for p in params[C.INTERFEROGRAM_FILES]
]
ifgs = common.small_data_setup()
correct._copy_mlooked(params)
tiles = pyrate.core.shared.get_tiles(cls.sampled_paths[0], rows, cols)
correct.correct_ifgs(config)
pyrate.main.timeseries(config)
pyrate.main.stack(config)
cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = \
(params[C.REFX], params[C.REFY]), params[C.MAXVAR], params[
C.VCMT]
cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles,
params[C.OUT_DIR])
cls.rate_p, cls.error_p, cls.samples_p = \
[common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[C.TMPDIR], t) for t in rate_types]
common.remove_tifs(params[C.OUT_DIR])
# now create the non parallel version
cls.tif_dir_s = Path(tempfile.mkdtemp())
params = manipulate_test_conf(gamma_conf, cls.tif_dir_s)
params[C.PROCESSES] = 4
params[C.PROCESSOR] = 1 # gamma
params[C.IFG_FILE_LIST] = os.path.join(common.GAMMA_SML_TEST_DIR,
'ifms_17')
params[C.PARALLEL] = 0
params[C.APS_CORRECTION] = 0
params[C.REFX], params[C.REFY] = -1, -1
output_conf_file = 'gamma.conf'
output_conf = cls.tif_dir_s.joinpath(output_conf_file).as_posix()
pyrate.configuration.write_config_file(params=params,
output_conf_file=output_conf)
config = Configuration(output_conf)
params = config.__dict__
common.sub_process_run(f"pyrate conv2tif -f {output_conf}")
common.sub_process_run(f"pyrate prepifg -f {output_conf}")
correct._copy_mlooked(params)
correct.correct_ifgs(config)
pyrate.main.timeseries(config)
pyrate.main.stack(config)
cls.refpixel, cls.maxvar, cls.vcmt = \
(params[C.REFX], params[C.REFY]), params[C.MAXVAR], params[
C.VCMT]
cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles,
params[C.OUT_DIR])
cls.rate, cls.error, cls.samples = \
[common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[C.TMPDIR], t) for t in rate_types]
cls.params = params
def tearDownClass(cls):
common.remove_tifs(cls.params[OBS_DIR])
common.remove_tifs(cls.params[OUT_DIR])
def teardown_class(cls):
shutil.rmtree(cls.temp_out_dir)
common.remove_tifs(
cf.get_config_params(common.TEST_CONF_ROIPAC)[cf.OBS_DIR])
def setUpClass(cls):
rate_types = ['stack_rate', 'stack_error', 'stack_samples']
cls.tif_dir = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_GAMMA
from pyrate.configuration import Configuration
# change the required params
params = Configuration(cls.test_conf).__dict__
params[cf.OBS_DIR] = common.SML_TEST_GAMMA
params[cf.PROCESSES] = 4
params[cf.PROCESSOR] = 1 # gamma
params[cf.IFG_FILE_LIST] = os.path.join(common.SML_TEST_GAMMA,
'ifms_17')
params[cf.OUT_DIR] = cls.tif_dir
params[cf.PARALLEL] = 1
params[cf.APS_CORRECTION] = False
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
rows, cols = params["rows"], params["cols"]
# xlks, ylks, crop = cf.transform_params(params)
# base_unw_paths need to be geotiffed by converttogeotif
# and multilooked by run_prepifg
base_unw_paths = list(cf.parse_namelist(params[cf.IFG_FILE_LIST]))
multi_paths = [
MultiplePaths(params[cf.OUT_DIR],
b,
ifglksx=params[cf.IFG_LKSX],
ifgcropopt=params[cf.IFG_CROP_OPT])
for b in base_unw_paths
]
# dest_paths are tifs that have been geotif converted and multilooked
cls.converted_paths = [b.converted_path for b in multi_paths]
cls.sampled_paths = [b.sampled_path for b in multi_paths]
from copy import copy
orig_params = copy(params)
conv2tif.main(params)
prepifg.main(orig_params)
tiles = pyrate.core.shared.get_tiles(cls.sampled_paths[0], rows, cols)
ifgs = common.small_data_setup()
params[cf.INTERFEROGRAM_FILES] = multi_paths
cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = process.process_ifgs(
cls.sampled_paths, params, rows, cols)
cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate_p, cls.error_p, cls.samples_p = \
[common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types]
common.remove_tifs(params[cf.OBS_DIR])
# now create the non parallel version
cls.tif_dir_s = tempfile.mkdtemp()
params[cf.PARALLEL] = 0
params[cf.PROCESSES] = 1
params[cf.OUT_DIR] = cls.tif_dir_s
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
multi_paths = [
MultiplePaths(params[cf.OUT_DIR],
b,
ifglksx=params[cf.IFG_LKSX],
ifgcropopt=params[cf.IFG_CROP_OPT])
for b in base_unw_paths
]
cls.converted_paths_s = [b.converted_path for b in multi_paths]
cls.sampled_paths_s = [b.sampled_path for b in multi_paths]
orig_params = copy(params)
conv2tif.main(params)
prepifg.main(orig_params)
params[cf.INTERFEROGRAM_FILES] = multi_paths
cls.refpixel, cls.maxvar, cls.vcmt = process.process_ifgs(
cls.sampled_paths_s, params, rows, cols)
cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate, cls.error, cls.samples = \
[common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types]
