def setUpClass(cls):
cls.tif_dir = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_GAMMA
# change the required params
cls.params = cf.get_config_params(cls.test_conf)
cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA
cls.params[cf.PROCESSOR] = 1 # gamma
file_list = list(cf.parse_namelist(os.path.join(common.SML_TEST_GAMMA,
'ifms_17')))
fd, cls.params[cf.IFG_FILE_LIST] = tempfile.mkstemp(suffix='.conf',
dir=cls.tif_dir)
os.close(fd)
# write a short filelist with only 3 gamma unws
with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp:
for f in file_list[:3]:
fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n')
cls.params[cf.OUT_DIR] = cls.tif_dir
cls.params[cf.PARALLEL] = 0
cls.params[cf.REF_EST_METHOD] = 1
cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
# base_unw_paths need to be geotiffed and multilooked by run_prepifg
cls.base_unw_paths = cf.original_ifg_paths(
cls.params[cf.IFG_FILE_LIST])
cls.base_unw_paths.append(common.SML_TEST_DEM_GAMMA)
xlks, ylks, crop = cf.transform_params(cls.params)
# dest_paths are tifs that have been geotif converted and multilooked
run_prepifg.gamma_prepifg(cls.base_unw_paths, cls.params)
cls.base_unw_paths.pop() # removed dem as we don't want it in ifgs
dest_paths = cf.get_dest_paths(
cls.base_unw_paths, crop, cls.params, xlks)
cls.ifgs = common.small_data_setup(datafiles=dest_paths)
def setUpClass(cls):
cls.tif_dir = tempfile.mkdtemp()
cls.tif_dir_method2 = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_ROIPAC
# change the required params
cls.params = cf.get_config_params(cls.test_conf)
cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA
cls.params[cf.PROCESSOR] = 1 # gamma
file_list = cf.parse_namelist(
os.path.join(common.SML_TEST_GAMMA, 'ifms_17'))
cls.params[cf.IFG_FILE_LIST] = tempfile.mktemp(dir=cls.tif_dir)
# write a short filelist with only 3 gamma unws
with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp:
for f in file_list[:2]:
fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n')
cls.params[cf.OUT_DIR] = cls.tif_dir
cls.params[cf.PARALLEL] = True
cls.params[cf.REF_EST_METHOD] = 1
cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
cls.params[cf.APS_INCIDENCE_MAP] = common.SML_TEST_INCIDENCE
# base_unw_paths need to be geotiffed and multilooked by run_prepifg
base_unw_paths = run_pyrate.original_ifg_paths(
cls.params[cf.IFG_FILE_LIST])
# add dem
base_unw_paths.append(common.SML_TEST_DEM_GAMMA)
# add incidence
base_unw_paths.append(common.SML_TEST_INCIDENCE)
xlks, ylks, crop = run_pyrate.transform_params(cls.params)
import copy
cls.params_method2 = copy.copy(cls.params)
cls.params_method2[cf.OUT_DIR] = cls.tif_dir_method2
cls.params_method2[cf.APS_METHOD] = 2
# dest_paths are tifs that have been geotif converted and multilooked
run_prepifg.gamma_prepifg(base_unw_paths, cls.params)
run_prepifg.gamma_prepifg(base_unw_paths, cls.params_method2)
# removed incidence as we don't want it in ifgs list
base_unw_paths.pop()
# removed dem as we don't want it in ifgs list
base_unw_paths.pop()
dest_paths = run_pyrate.get_dest_paths(base_unw_paths, crop,
cls.params, xlks)
cls.ifgs = common.small_data_setup(datafiles=dest_paths)
dest_paths_m2 = run_pyrate.get_dest_paths(base_unw_paths, crop,
cls.params_method2, xlks)
cls.ifgs_method2 = common.small_data_setup(datafiles=dest_paths_m2)
aps.remove_aps_delay(cls.ifgs, cls.params)
aps.remove_aps_delay(cls.ifgs_method2, cls.params_method2)
def setUpClass(cls):
rate_types = ['linrate', 'linerror', 'linsamples']
cls.tif_dir = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_GAMMA
# change the required params
params = cf.get_config_params(cls.test_conf)
params[cf.OBS_DIR] = common.SML_TEST_GAMMA
params[cf.PROCESSOR] = 1 # gamma
params[cf.IFG_FILE_LIST] = os.path.join(common.SML_TEST_GAMMA,
'ifms_17')
params[cf.OUT_DIR] = cls.tif_dir
params[cf.PARALLEL] = 0
params[cf.APS_CORRECTION] = False
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
xlks, ylks, crop = cf.transform_params(params)
# base_unw_paths need to be geotiffed and multilooked by run_prepifg
base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
# dest_paths are tifs that have been geotif converted and multilooked
cls.dest_paths = cf.get_dest_paths(base_unw_paths, crop, params, xlks)
run_prepifg.gamma_prepifg(base_unw_paths, params)
tiles = run_pyrate.get_tiles(cls.dest_paths[0], 3, 3)
ifgs = common.small_data_setup()
cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = \
run_pyrate.process_ifgs(cls.dest_paths, params, 3, 3)
cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate_p, cls.error_p, cls.samples_p = [
common.reconstruct_linrate(ifgs[0].shape, tiles, params[cf.TMPDIR],
t) for t in rate_types
]
# now create the non parallel version
cls.tif_dir_s = tempfile.mkdtemp()
params[cf.PARALLEL] = 0
params[cf.OUT_DIR] = cls.tif_dir_s
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
cls.dest_paths_s = cf.get_dest_paths(base_unw_paths, crop, params,
xlks)
run_prepifg.gamma_prepifg(base_unw_paths, params)
cls.refpixel, cls.maxvar, cls.vcmt = \
run_pyrate.process_ifgs(cls.dest_paths_s, params, 3, 3)
cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate, cls.error, cls.samples = [
common.reconstruct_linrate(ifgs[0].shape, tiles, params[cf.TMPDIR],
t) for t in rate_types
]
def test_prepifg_mpi(mpisync, get_config, tempdir, roipac_or_gamma, get_lks,
get_crop):
from tests.common import TEST_CONF_ROIPAC, TEST_CONF_GAMMA
from os.path import join, basename
if roipac_or_gamma == 1:
params = get_config(TEST_CONF_GAMMA)
else:
params = get_config(TEST_CONF_ROIPAC)
outdir = mpiops.run_once(tempdir)
params[cf.OUT_DIR] = outdir
params[cf.PARALLEL] = False
params[cf.IFG_LKSX], params[cf.IFG_LKSY] = get_lks, get_lks
params[cf.IFG_CROP_OPT] = get_crop
if roipac_or_gamma == 1:
params[cf.IFG_FILE_LIST] = join(common.SML_TEST_GAMMA, 'ifms_17')
params[cf.OBS_DIR] = common.SML_TEST_GAMMA
params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
params[cf.DEM_HEADER_FILE] = common.SML_TEST_DEM_HDR_GAMMA
run_prepifg.main(params)
if mpiops.rank == 0:
if roipac_or_gamma == 1:
params_s = get_config(TEST_CONF_GAMMA)
else:
params_s = get_config(TEST_CONF_ROIPAC)
params_s[cf.OUT_DIR] = tempdir()
params_s[cf.PARALLEL] = True
params_s[cf.IFG_LKSX], params_s[cf.IFG_LKSY] = get_lks, get_lks
params_s[cf.IFG_CROP_OPT] = get_crop
if roipac_or_gamma == 1:
base_unw_paths = glob.glob(join(common.SML_TEST_GAMMA,
"*_utm.unw"))
run_prepifg.gamma_prepifg(base_unw_paths, params_s)
else:
base_unw_paths = glob.glob(join(common.SML_TEST_OBS, "*.unw"))
run_prepifg.roipac_prepifg(base_unw_paths, params_s)
mpi_tifs = glob.glob(join(outdir, "*.tif"))
serial_tifs = glob.glob(join(params[cf.OUT_DIR], "*.tif"))
mpi_tifs.sort()
serial_tifs.sort()
# 17 geotifs, and 17 mlooked tifs
assert len(mpi_tifs) == len(serial_tifs)
for m_f, s_f in zip(mpi_tifs, serial_tifs):
assert basename(m_f) == basename(s_f)
shutil.rmtree(outdir)
shutil.rmtree(params_s[cf.OUT_DIR])
def setUpClass(cls):
cls.serial_dir = tempfile.mkdtemp()
cls.parallel_dir = tempfile.mkdtemp()
unw_paths = glob.glob(os.path.join(SML_TEST_GAMMA, "*_utm.unw"))
# read in the params
_, _, params = cf.get_ifg_paths(TEST_CONF_GAMMA)
params[cf.OUT_DIR] = cls.serial_dir
params[cf.PARALLEL] = False
shared.mkdir_p(cls.serial_dir)
run_prepifg.gamma_prepifg(unw_paths, params)
params[cf.OUT_DIR] = cls.parallel_dir
params[cf.PARALLEL] = True
shared.mkdir_p(cls.parallel_dir)
run_prepifg.gamma_prepifg(unw_paths, params)
def test_timeseries_linrate_mpi(mpisync, tempdir, modify_config,
ref_est_method, row_splits, col_splits,
get_crop, orbfit_lks, orbfit_method,
orbfit_degrees):
params = modify_config
outdir = mpiops.run_once(tempdir)
params[cf.OUT_DIR] = outdir
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
params[cf.DEM_HEADER_FILE] = SML_TEST_DEM_HDR_GAMMA
params[cf.REF_EST_METHOD] = ref_est_method
params[cf.IFG_CROP_OPT] = get_crop
params[cf.ORBITAL_FIT_LOOKS_Y] = orbfit_lks
params[cf.ORBITAL_FIT_LOOKS_X] = orbfit_lks
params[cf.ORBITAL_FIT_METHOD] = orbfit_method
params[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees
xlks, ylks, crop = cf.transform_params(params)
if xlks * col_splits > 45 or ylks * row_splits > 70:
print('skipping test because lks and col_splits are not compatible')
return
# skip some tests in travis to run CI faster
if TRAVIS and (xlks % 2 or row_splits % 2 or col_splits % 2
or orbfit_lks % 2):
print('Skipping in travis env for faster CI run')
return
print("xlks={}, ref_est_method={}, row_splits={}, col_splits={}, "
"get_crop={}, orbfit_lks={}, orbfit_method={}, "
"rank={}".format(xlks, ref_est_method, row_splits, col_splits,
get_crop, orbfit_lks, orbfit_method, orbfit_degrees,
mpiops.rank))
base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
# dest_paths are tifs that have been geotif converted and multilooked
dest_paths = cf.get_dest_paths(base_unw_paths, crop, params, xlks)
# run prepifg, create the dest_paths files
if mpiops.rank == 0:
run_prepifg.gamma_prepifg(base_unw_paths, params)
mpiops.comm.barrier()
(refpx,
refpy), maxvar, vcmt = run_pyrate.process_ifgs(ifg_paths=dest_paths,
params=params,
rows=row_splits,
cols=col_splits)
tiles = mpiops.run_once(run_pyrate.get_tiles,
dest_paths[0],
rows=row_splits,
cols=col_splits)
postprocessing.postprocess_linrate(row_splits, col_splits, params)
postprocessing.postprocess_timeseries(row_splits, col_splits, params)
ifgs_mpi_out_dir = params[cf.OUT_DIR]
ifgs_mpi = small_data_setup(datafiles=dest_paths)
# single process timeseries/linrate calculation
if mpiops.rank == 0:
params_old = modify_config
params_old[cf.OUT_DIR] = tempdir()
params_old[cf.REF_EST_METHOD] = ref_est_method
params_old[cf.IFG_CROP_OPT] = get_crop
params_old[cf.ORBITAL_FIT_LOOKS_Y] = orbfit_lks
params_old[cf.ORBITAL_FIT_LOOKS_X] = orbfit_lks
params_old[cf.ORBITAL_FIT_METHOD] = orbfit_method
params_old[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees
xlks, ylks, crop = cf.transform_params(params_old)
base_unw_paths = cf.original_ifg_paths(params_old[cf.IFG_FILE_LIST])
dest_paths = cf.get_dest_paths(base_unw_paths, crop, params_old, xlks)
run_prepifg.gamma_prepifg(base_unw_paths, params_old)
ifgs = shared.pre_prepare_ifgs(dest_paths, params_old)
mst_grid = tests.common.mst_calculation(dest_paths, params_old)
refy, refx = refpixel.ref_pixel(ifgs, params_old)
assert (refx == refpx) and (refy == refpy) # both must match
pyrate.orbital.remove_orbital_error(ifgs, params_old)
ifgs = common.prepare_ifgs_without_phase(dest_paths, params_old)
rpe.estimate_ref_phase(ifgs, params_old, refx, refy)
ifgs = shared.pre_prepare_ifgs(dest_paths, params_old)
maxvar_s = [vcm.cvd(i, params_old)[0] for i in ifgs]
vcmt_s = vcm.get_vcmt(ifgs, maxvar)
tsincr, tscum, _ = tests.common.compute_time_series(
ifgs, mst_grid, params, vcmt)
rate, error, samples = tests.common.calculate_linear_rate(
ifgs, params_old, vcmt, mst_grid)
mst_mpi = reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR])
np.testing.assert_array_almost_equal(mst_grid, mst_mpi)
tsincr_mpi, tscum_mpi = reconstruct_times_series(
ifgs[0].shape, tiles, params[cf.TMPDIR])
rate_mpi, error_mpi, samples_mpi = \
[reconstruct_linrate(ifgs[0].shape, tiles, params[cf.TMPDIR], t)
for t in ['linrate', 'linerror', 'linsamples']]
np.testing.assert_array_almost_equal(maxvar, maxvar_s)
np.testing.assert_array_almost_equal(vcmt, vcmt_s)
for i, j in zip(ifgs, ifgs_mpi):
np.testing.assert_array_almost_equal(i.phase_data, j.phase_data)
np.testing.assert_array_almost_equal(tsincr, tsincr_mpi, decimal=4)
np.testing.assert_array_almost_equal(tscum, tscum_mpi, decimal=4)
np.testing.assert_array_almost_equal(rate, rate_mpi, decimal=4)
np.testing.assert_array_almost_equal(error, error_mpi, decimal=4)
np.testing.assert_array_almost_equal(samples, samples_mpi, decimal=4)
# assert linear rate output tifs are same
_tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'linrate.tif')
_tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'linerror.tif')
_tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'linsamples.tif')
# assert time series output tifs are same
epochlist = algorithm.get_epochs(ifgs)[0]
for i in range(tsincr.shape[2]):
_tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR],
'tsincr' + '_' + str(epochlist.dates[i + 1]) + ".tif")
# 12 timeseries outputs
assert i + 1 == tsincr.shape[2]
shutil.rmtree(ifgs_mpi_out_dir) # remove mpi out dir
shutil.rmtree(params_old[cf.OUT_DIR]) # remove serial out dir
def setUpClass(cls):
cls.tif_dir_serial = tempfile.mkdtemp()
cls.tif_dir_mpi = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_ROIPAC
# change the required params
cls.params = cf.get_config_params(cls.test_conf)
cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA
cls.params[cf.PROCESSOR] = 1 # gamma
file_list = cf.parse_namelist(
os.path.join(common.SML_TEST_GAMMA, 'ifms_17'))
cls.params[cf.IFG_FILE_LIST] = tempfile.mktemp(dir=cls.tif_dir_serial)
# write a short filelist with only 3 gamma unws
with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp:
for f in file_list[:2]:
fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n')
cls.params[cf.OUT_DIR] = cls.tif_dir_serial
cls.params[cf.PARALLEL] = 0
cls.params[cf.REF_EST_METHOD] = 2
cls.params[cf.IFG_LKSX] = 1
cls.params[cf.IFG_LKSY] = 1
cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
cls.params[cf.APS_INCIDENCE_MAP] = common.SML_TEST_INCIDENCE
# base_unw_paths need to be geotiffed and multilooked by run_prepifg
base_unw_paths = run_pyrate.original_ifg_paths(
cls.params[cf.IFG_FILE_LIST])
# add dem
base_unw_paths.append(common.SML_TEST_DEM_GAMMA)
# add incidence
base_unw_paths.append(common.SML_TEST_INCIDENCE)
xlks, ylks, crop = run_pyrate.transform_params(cls.params)
cls.params_mpi = copy.copy(cls.params)
cls.params_mpi[cf.OUT_DIR] = cls.tif_dir_mpi
# dest_paths are tifs that have been geotif converted and multilooked
run_prepifg.gamma_prepifg(base_unw_paths, cls.params)
run_prepifg.gamma_prepifg(base_unw_paths, cls.params_mpi)
# removed incidence as we don't want it in ifgs list
base_unw_paths.pop()
# removed dem as we don't want it in ifgs list
base_unw_paths.pop()
dest_paths = run_pyrate.get_dest_paths(base_unw_paths, crop,
cls.params, xlks)
dest_paths_mpi = run_pyrate.get_dest_paths(base_unw_paths, crop,
cls.params_mpi, xlks)
run_pyrate.process_ifgs(dest_paths, cls.params)
cls.ifgs_serial = common.small_data_setup(datafiles=dest_paths)
cls.conf_mpi = tempfile.mktemp('.conf', dir=cls.tif_dir_mpi)
cf.write_config_file(cls.params_mpi, cls.conf_mpi)
str = 'mpirun -np 4 python pyrate/nci/run_pyrate_pypar.py ' + \
cls.conf_mpi
cmd = str.split()
subprocess.check_call(cmd)
str = 'mpirun -np 4 python pyrate/nci/run_pyrate_pypar_2.py ' + \
cls.conf_mpi
cmd = str.split()
subprocess.check_call(cmd)
str = 'mpirun -np 4 python pyrate/nci/run_pyrate_pypar_3.py ' + \
cls.conf_mpi
cmd = str.split()
subprocess.check_call(cmd)
cls.ifgs_mpi = common.small_data_setup(datafiles=dest_paths_mpi)
