def postprocess_linrate(rows, cols, params):
"""
Postprocess linear rate.
:param rows: xxxx
:param cols: xxxx
:param params: xxxx
:return xxxx
"""
# pylint: disable=expression-not-assigned
# setup paths
xlks, _, crop = cf.transform_params(params)
base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
dest_tifs = cf.get_dest_paths(base_unw_paths, crop, params, xlks)
# load previously saved prepread_ifgs dict
preread_ifgs_file = join(params[cf.TMPDIR], 'preread_ifgs.pk')
ifgs = cp.load(open(preread_ifgs_file, 'rb'))
tiles = run_pyrate.get_tiles(dest_tifs[0], rows, cols)
# linrate aggregation
if mpiops.size >= 3:
[
save_linrate(ifgs, params, tiles, out_type=t)
for i, t in enumerate(['linrate', 'linerror', 'linsamples'])
if i == mpiops.rank
]
else:
if mpiops.rank == MASTER_PROCESS:
[
save_linrate(ifgs, params, tiles, out_type=t)
for t in ['linrate', 'linerror', 'linsamples']
]
def roipac_prepifg(base_ifg_paths, params):
"""
Prepare ROI_PAC interferograms which combines both conversion to geotiff
and multilooking/cropping operations.
:param list base_ifg_paths: List of unwrapped interferograms
:param dict params: Parameters dictionary corresponding to config file
"""
log.info("Preparing ROI_PAC format interferograms")
parallel = params[cf.PARALLEL]
if parallel:
log.info("Parallel prepifg is not implemented for ROI_PAC")
log.info("Running prepifg in serial")
xlooks, ylooks, crop = cf.transform_params(params)
rsc_file = os.path.join(params[cf.DEM_HEADER_FILE])
if rsc_file is not None:
projection = roipac.parse_header(rsc_file)[ifc.PYRATE_DATUM]
else:
raise roipac.RoipacException('No DEM resource/header file is '
'provided')
dest_base_ifgs = [os.path.join(params[cf.OUT_DIR],
os.path.basename(q).split('.')[0] + '_' +
os.path.basename(q).split('.')[1] + '.tif')
for q in base_ifg_paths]
for b, d in zip(base_ifg_paths, dest_base_ifgs):
header_file = "%s.%s" % (b, ROI_PAC_HEADER_FILE_EXT)
header = roipac.manage_header(header_file, projection)
write_geotiff(header, b, d, nodata=params[cf.NO_DATA_VALUE])
prepifg.prepare_ifgs(
dest_base_ifgs, crop_opt=crop, xlooks=xlooks, ylooks=ylooks)
def setUpClass(cls):
cls.tif_dir = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_GAMMA
# change the required params
cls.params = cf.get_config_params(cls.test_conf)
cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA
cls.params[cf.PROCESSOR] = 1 # gamma
file_list = list(cf.parse_namelist(os.path.join(common.SML_TEST_GAMMA,
'ifms_17')))
fd, cls.params[cf.IFG_FILE_LIST] = tempfile.mkstemp(suffix='.conf',
dir=cls.tif_dir)
os.close(fd)
# write a short filelist with only 3 gamma unws
with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp:
for f in file_list[:3]:
fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n')
cls.params[cf.OUT_DIR] = cls.tif_dir
cls.params[cf.PARALLEL] = 0
cls.params[cf.REF_EST_METHOD] = 1
cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
# base_unw_paths need to be geotiffed and multilooked by run_prepifg
cls.base_unw_paths = cf.original_ifg_paths(
cls.params[cf.IFG_FILE_LIST])
cls.base_unw_paths.append(common.SML_TEST_DEM_GAMMA)
xlks, ylks, crop = cf.transform_params(cls.params)
# dest_paths are tifs that have been geotif converted and multilooked
run_prepifg.gamma_prepifg(cls.base_unw_paths, cls.params)
cls.base_unw_paths.pop() # removed dem as we don't want it in ifgs
dest_paths = cf.get_dest_paths(
cls.base_unw_paths, crop, cls.params, xlks)
cls.ifgs = common.small_data_setup(datafiles=dest_paths)
def setUpClass(cls):
params = cf.get_config_params(common.TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['run_prepifg.py', common.TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
run_prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
# start run_pyrate copy
ifgs = common.pre_prepare_ifgs(dest_paths, params)
mst_grid = common.mst_calculation(dest_paths, params)
refx, refy = run_pyrate._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.orbital.remove_orbital_error(ifgs, params)
ifgs = common.prepare_ifgs_without_phase(dest_paths, params)
_, ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy)
r_dist = covariance.RDist(ifgs[0])()
maxvar = [covariance.cvd(i, params, r_dist)[0] for i in ifgs]
vcmt = covariance.get_vcmt(ifgs, maxvar)
params[cf.TIME_SERIES_METHOD] = 1
params[cf.PARALLEL] = 0
# Calculate time series
cls.tsincr_0, cls.tscum_0, _ = common.calculate_time_series(
ifgs, params, vcmt, mst=mst_grid)
params[cf.PARALLEL] = 1
cls.tsincr_1, cls.tscum_1, cls.tsvel_1 = \
common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid)
params[cf.PARALLEL] = 2
cls.tsincr_2, cls.tscum_2, cls.tsvel_2 = \
common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid)
# load the matlab data
ts_dir = os.path.join(common.SML_TEST_DIR, 'matlab_time_series')
tsincr_path = os.path.join(ts_dir, 'ts_incr_interp0_method1.csv')
ts_incr = np.genfromtxt(tsincr_path)
# the matlab tsvel return is a bit pointless and not tested here
# tserror is not returned
# tserr_path = os.path.join(SML_TIME_SERIES_DIR,
# 'ts_error_interp0_method1.csv')
# ts_err = np.genfromtxt(tserr_path, delimiter=',')
tscum_path = os.path.join(ts_dir, 'ts_cum_interp0_method1.csv')
ts_cum = np.genfromtxt(tscum_path)
cls.ts_incr = np.reshape(ts_incr,
newshape=cls.tsincr_0.shape,
order='F')
cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F')
def setUpClass(cls):
rate_types = ['linrate', 'linerror', 'linsamples']
cls.tif_dir = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_GAMMA
# change the required params
params = cf.get_config_params(cls.test_conf)
params[cf.OBS_DIR] = common.SML_TEST_GAMMA
params[cf.PROCESSOR] = 1 # gamma
params[cf.IFG_FILE_LIST] = os.path.join(common.SML_TEST_GAMMA,
'ifms_17')
params[cf.OUT_DIR] = cls.tif_dir
params[cf.PARALLEL] = 0
params[cf.APS_CORRECTION] = False
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
xlks, ylks, crop = cf.transform_params(params)
# base_unw_paths need to be geotiffed and multilooked by run_prepifg
base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
# dest_paths are tifs that have been geotif converted and multilooked
cls.dest_paths = cf.get_dest_paths(base_unw_paths, crop, params, xlks)
run_prepifg.gamma_prepifg(base_unw_paths, params)
tiles = run_pyrate.get_tiles(cls.dest_paths[0], 3, 3)
ifgs = common.small_data_setup()
cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = \
run_pyrate.process_ifgs(cls.dest_paths, params, 3, 3)
cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate_p, cls.error_p, cls.samples_p = [
common.reconstruct_linrate(ifgs[0].shape, tiles, params[cf.TMPDIR],
t) for t in rate_types
]
# now create the non parallel version
cls.tif_dir_s = tempfile.mkdtemp()
params[cf.PARALLEL] = 0
params[cf.OUT_DIR] = cls.tif_dir_s
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
cls.dest_paths_s = cf.get_dest_paths(base_unw_paths, crop, params,
xlks)
run_prepifg.gamma_prepifg(base_unw_paths, params)
cls.refpixel, cls.maxvar, cls.vcmt = \
run_pyrate.process_ifgs(cls.dest_paths_s, params, 3, 3)
cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate, cls.error, cls.samples = [
common.reconstruct_linrate(ifgs[0].shape, tiles, params[cf.TMPDIR],
t) for t in rate_types
]
def gamma_prepifg(base_unw_paths, params):
"""
GAMMA prepifg which combines both conversion to geotiff and multi-looking
and cropping.
:param base_unw_paths: List of unwrapped interferograms
:param params: Parameters dictionary corresponding to config file
:return xxxx
"""
# pylint: disable=expression-not-assigned
log.info("Preparing GAMMA format interferograms")
parallel = params[cf.PARALLEL]
# dest_base_ifgs: location of geo_tif's
if parallel:
log.info("Running prepifg in parallel with {} "
"processes".format(params[cf.PROCESSES]))
dest_base_ifgs = Parallel(n_jobs=params[cf.PROCESSES], verbose=50)(
delayed(gamma_multiprocessing)(p, params) for p in base_unw_paths)
else:
log.info("Running prepifg in serial")
dest_base_ifgs = [
gamma_multiprocessing(b, params) for b in base_unw_paths
]
ifgs = [prepifg.dem_or_ifg(p) for p in dest_base_ifgs]
xlooks, ylooks, crop = cf.transform_params(params)
user_exts = (params[cf.IFG_XFIRST], params[cf.IFG_YFIRST],
params[cf.IFG_XLAST], params[cf.IFG_YLAST])
exts = prepifg.get_analysis_extent(crop,
ifgs,
xlooks,
ylooks,
user_exts=user_exts)
thresh = params[cf.NO_DATA_AVERAGING_THRESHOLD]
if parallel:
Parallel(n_jobs=params[cf.PROCESSES], verbose=50)(
delayed(prepifg.prepare_ifg)(p, xlooks, ylooks, exts, thresh, crop)
for p in dest_base_ifgs)
else:
[
prepifg.prepare_ifg(i, xlooks, ylooks, exts, thresh, crop)
for i in dest_base_ifgs
]
def setUpClass(cls):
params = cf.get_config_params(TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['run_prepifg.py', TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.REF_EST_METHOD] = 2
run_prepifg.main(params)
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
ifgs = shared.pre_prepare_ifgs(dest_paths, params)
refx, refy = run_pyrate.ref_pixel_calc(dest_paths, params)
pyrate.orbital.remove_orbital_error(ifgs, params)
ifgs = prepare_ifgs_without_phase(dest_paths, params)
_, ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy)
# Calculate interferogram noise
cls.maxvar = [cvd(i, params)[0] for i in ifgs]
cls.vcmt = get_vcmt(ifgs, cls.maxvar)
def setUpClass(cls):
params = cf.get_config_params(common.TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['run_prepifg.py', common.TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
run_prepifg.main(params)
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.REF_EST_METHOD] = 2
params[cf.PARALLEL] = False
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop,
params, xlks)
# start run_pyrate copy
ifgs = shared.pre_prepare_ifgs(dest_paths, params)
mst_grid = tests.common.mst_calculation(dest_paths, params)
# Estimate reference pixel location
refx, refy = run_pyrate.ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.orbital.remove_orbital_error(ifgs, params)
for i in ifgs:
i.close()
ifgs = shared.pre_prepare_ifgs(dest_paths, params)
cls.ref_phs, cls.ifgs = estimate_ref_phase(ifgs, params, refx, refy)
# end run_pyrate copy
for i in ifgs:
i.close()
def test_vcm_matlab_vs_mpi(mpisync, tempdir, get_config):
from tests.common import SML_TEST_DIR, TEST_CONF_ROIPAC
params_dict = get_config(TEST_CONF_ROIPAC)
MATLAB_VCM_DIR = os.path.join(SML_TEST_DIR, 'matlab_vcm')
matlab_vcm = np.genfromtxt(os.path.join(MATLAB_VCM_DIR, 'matlab_vcmt.csv'),
delimiter=',')
if mpiops.rank == 0:
outdir = tempdir()
else:
outdir = None
outdir = mpiops.comm.bcast(outdir, root=0)
params_dict[cf.OUT_DIR] = outdir
params_dict[cf.PARALLEL] = False
xlks, ylks, crop = cf.transform_params(params_dict)
base_unw_paths = cf.original_ifg_paths(params_dict[cf.IFG_FILE_LIST])
# dest_paths are tifs that have been geotif converted and multilooked
dest_paths = cf.get_dest_paths(base_unw_paths, crop, params_dict, xlks)
# run prepifg, create the dest_paths files
if mpiops.rank == 0:
run_prepifg.roipac_prepifg(base_unw_paths, params_dict)
mpiops.comm.barrier()
tiles = run_pyrate.get_tiles(dest_paths[0], rows=1, cols=1)
preread_ifgs = run_pyrate.create_ifg_dict(dest_paths,
params=params_dict,
tiles=tiles)
refpx, refpy = run_pyrate.ref_pixel_calc(dest_paths, params_dict)
run_pyrate.orb_fit_calc(dest_paths, params_dict)
run_pyrate.ref_phase_estimation(dest_paths, params_dict, refpx, refpy)
maxvar, vcmt = run_pyrate.maxvar_vcm_calc(dest_paths, params_dict,
preread_ifgs)
np.testing.assert_array_almost_equal(maxvar, matlab_maxvar, decimal=4)
np.testing.assert_array_almost_equal(matlab_vcm, vcmt, decimal=3)
if mpiops.rank == 0:
shutil.rmtree(outdir)
def setUpClass(cls):
params = cf.get_config_params(TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['run_prepifg.py', TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.TMPDIR] = os.path.join(cls.temp_out_dir, cf.TMPDIR)
shared.mkdir_p(params[cf.TMPDIR])
params[cf.REF_EST_METHOD] = 2
run_prepifg.main(params)
cls.params = params
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
ifgs = common.pre_prepare_ifgs(dest_paths, params)
refx, refy = run_pyrate._ref_pixel_calc(dest_paths, params)
pyrate.orbital.remove_orbital_error(ifgs, params)
ifgs = prepare_ifgs_without_phase(dest_paths, params)
_, cls.ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy)
r_dist = RDist(ifgs[0])()
# Calculate interferogram noise
cls.maxvar = [cvd(i, params, r_dist, calc_alpha=True,
save_acg=True, write_vals=True)[0] for i in ifgs]
cls.vcmt = get_vcmt(ifgs, cls.maxvar)
def setUpClass(cls):
params = cf.get_config_params(TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['run_prepifg.py', TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
shared.mkdir_p(params[cf.TMPDIR])
run_prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
xlks, _, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
# start run_pyrate copy
ifgs = pre_prepare_ifgs(dest_paths, params)
mst_grid = tests.common.mst_calculation(dest_paths, params)
refx, refy = run_pyrate._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.orbital.remove_orbital_error(ifgs, params)
ifgs = prepare_ifgs_without_phase(dest_paths, params)
_, ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy)
r_dist = vcm_module.RDist(ifgs[0])()
maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in ifgs]
vcmt = vcm_module.get_vcmt(ifgs, maxvar)
# Calculate linear rate map
params[cf.PARALLEL] = 1
cls.rate, cls.error, cls.samples = tests.common.calculate_linear_rate(
ifgs, params, vcmt, mst_mat=mst_grid)
params[cf.PARALLEL] = 2
cls.rate_2, cls.error_2, cls.samples_2 = \
tests.common.calculate_linear_rate(ifgs, params, vcmt,
mst_mat=mst_grid)
params[cf.PARALLEL] = 0
# Calculate linear rate map
cls.rate_s, cls.error_s, cls.samples_s = \
tests.common.calculate_linear_rate(ifgs, params, vcmt,
mst_mat=mst_grid)
matlab_linrate_dir = os.path.join(SML_TEST_DIR, 'matlab_linrate')
cls.rate_matlab = np.genfromtxt(os.path.join(matlab_linrate_dir,
'stackmap.csv'),
delimiter=',')
cls.error_matlab = np.genfromtxt(os.path.join(matlab_linrate_dir,
'errormap.csv'),
delimiter=',')
cls.samples_matlab = np.genfromtxt(os.path.join(
matlab_linrate_dir, 'coh_sta.csv'),
delimiter=',')
def test_timeseries_linrate_mpi(mpisync, tempdir, modify_config,
ref_est_method, row_splits, col_splits,
get_crop, orbfit_lks, orbfit_method,
orbfit_degrees):
params = modify_config
outdir = mpiops.run_once(tempdir)
params[cf.OUT_DIR] = outdir
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
params[cf.DEM_HEADER_FILE] = SML_TEST_DEM_HDR_GAMMA
params[cf.REF_EST_METHOD] = ref_est_method
params[cf.IFG_CROP_OPT] = get_crop
params[cf.ORBITAL_FIT_LOOKS_Y] = orbfit_lks
params[cf.ORBITAL_FIT_LOOKS_X] = orbfit_lks
params[cf.ORBITAL_FIT_METHOD] = orbfit_method
params[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees
xlks, ylks, crop = cf.transform_params(params)
if xlks * col_splits > 45 or ylks * row_splits > 70:
print('skipping test because lks and col_splits are not compatible')
return
# skip some tests in travis to run CI faster
if TRAVIS and (xlks % 2 or row_splits % 2 or col_splits % 2
or orbfit_lks % 2):
print('Skipping in travis env for faster CI run')
return
print("xlks={}, ref_est_method={}, row_splits={}, col_splits={}, "
"get_crop={}, orbfit_lks={}, orbfit_method={}, "
"rank={}".format(xlks, ref_est_method, row_splits, col_splits,
get_crop, orbfit_lks, orbfit_method, orbfit_degrees,
mpiops.rank))
base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
# dest_paths are tifs that have been geotif converted and multilooked
dest_paths = cf.get_dest_paths(base_unw_paths, crop, params, xlks)
# run prepifg, create the dest_paths files
if mpiops.rank == 0:
run_prepifg.gamma_prepifg(base_unw_paths, params)
mpiops.comm.barrier()
(refpx,
refpy), maxvar, vcmt = run_pyrate.process_ifgs(ifg_paths=dest_paths,
params=params,
rows=row_splits,
cols=col_splits)
tiles = mpiops.run_once(run_pyrate.get_tiles,
dest_paths[0],
rows=row_splits,
cols=col_splits)
postprocessing.postprocess_linrate(row_splits, col_splits, params)
postprocessing.postprocess_timeseries(row_splits, col_splits, params)
ifgs_mpi_out_dir = params[cf.OUT_DIR]
ifgs_mpi = small_data_setup(datafiles=dest_paths)
# single process timeseries/linrate calculation
if mpiops.rank == 0:
params_old = modify_config
params_old[cf.OUT_DIR] = tempdir()
params_old[cf.REF_EST_METHOD] = ref_est_method
params_old[cf.IFG_CROP_OPT] = get_crop
params_old[cf.ORBITAL_FIT_LOOKS_Y] = orbfit_lks
params_old[cf.ORBITAL_FIT_LOOKS_X] = orbfit_lks
params_old[cf.ORBITAL_FIT_METHOD] = orbfit_method
params_old[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees
xlks, ylks, crop = cf.transform_params(params_old)
base_unw_paths = cf.original_ifg_paths(params_old[cf.IFG_FILE_LIST])
dest_paths = cf.get_dest_paths(base_unw_paths, crop, params_old, xlks)
run_prepifg.gamma_prepifg(base_unw_paths, params_old)
ifgs = shared.pre_prepare_ifgs(dest_paths, params_old)
mst_grid = tests.common.mst_calculation(dest_paths, params_old)
refy, refx = refpixel.ref_pixel(ifgs, params_old)
assert (refx == refpx) and (refy == refpy) # both must match
pyrate.orbital.remove_orbital_error(ifgs, params_old)
ifgs = common.prepare_ifgs_without_phase(dest_paths, params_old)
rpe.estimate_ref_phase(ifgs, params_old, refx, refy)
ifgs = shared.pre_prepare_ifgs(dest_paths, params_old)
maxvar_s = [vcm.cvd(i, params_old)[0] for i in ifgs]
vcmt_s = vcm.get_vcmt(ifgs, maxvar)
tsincr, tscum, _ = tests.common.compute_time_series(
ifgs, mst_grid, params, vcmt)
rate, error, samples = tests.common.calculate_linear_rate(
ifgs, params_old, vcmt, mst_grid)
mst_mpi = reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR])
np.testing.assert_array_almost_equal(mst_grid, mst_mpi)
tsincr_mpi, tscum_mpi = reconstruct_times_series(
ifgs[0].shape, tiles, params[cf.TMPDIR])
rate_mpi, error_mpi, samples_mpi = \
[reconstruct_linrate(ifgs[0].shape, tiles, params[cf.TMPDIR], t)
for t in ['linrate', 'linerror', 'linsamples']]
np.testing.assert_array_almost_equal(maxvar, maxvar_s)
np.testing.assert_array_almost_equal(vcmt, vcmt_s)
for i, j in zip(ifgs, ifgs_mpi):
np.testing.assert_array_almost_equal(i.phase_data, j.phase_data)
np.testing.assert_array_almost_equal(tsincr, tsincr_mpi, decimal=4)
np.testing.assert_array_almost_equal(tscum, tscum_mpi, decimal=4)
np.testing.assert_array_almost_equal(rate, rate_mpi, decimal=4)
np.testing.assert_array_almost_equal(error, error_mpi, decimal=4)
np.testing.assert_array_almost_equal(samples, samples_mpi, decimal=4)
# assert linear rate output tifs are same
_tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'linrate.tif')
_tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'linerror.tif')
_tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'linsamples.tif')
# assert time series output tifs are same
epochlist = algorithm.get_epochs(ifgs)[0]
for i in range(tsincr.shape[2]):
_tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR],
'tsincr' + '_' + str(epochlist.dates[i + 1]) + ".tif")
# 12 timeseries outputs
assert i + 1 == tsincr.shape[2]
shutil.rmtree(ifgs_mpi_out_dir) # remove mpi out dir
shutil.rmtree(params_old[cf.OUT_DIR]) # remove serial out dir
def postprocess_timeseries(rows, cols, params):
"""
Postprocess time series output.
:param rows: xxxx
:param cols: xxxx
:param params: xxxx
:return xxxx
"""
# pylint: disable=too-many-locals
xlks, _, crop = cf.transform_params(params)
base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST])
dest_tifs = cf.get_dest_paths(base_unw_paths, crop, params, xlks)
output_dir = params[cf.TMPDIR]
# load previously saved prepread_ifgs dict
preread_ifgs_file = join(output_dir, 'preread_ifgs.pk')
ifgs = cp.load(open(preread_ifgs_file, 'rb'))
# metadata and projections
gt, md, wkt = ifgs['gt'], ifgs['md'], ifgs['wkt']
epochlist = ifgs['epochlist']
ifgs = [v for v in ifgs.values() if isinstance(v, PrereadIfg)]
tiles = run_pyrate.get_tiles(dest_tifs[0], rows, cols)
# load the first tsincr file to determine the number of time series tifs
tsincr_file = os.path.join(output_dir, 'tsincr_0.npy')
tsincr = np.load(file=tsincr_file)
# pylint: disable=no-member
no_ts_tifs = tsincr.shape[2]
# we create 2 x no_ts_tifs as we are splitting tsincr and tscuml
# to all processes.
process_tifs = mpiops.array_split(range(2 * no_ts_tifs))
# depending on nvelpar, this will not fit in memory
# e.g. nvelpar=100, nrows=10000, ncols=10000, 32bit floats need 40GB memory
# 32 * 100 * 10000 * 10000 / 8 bytes = 4e10 bytes = 40 GB
# the double for loop helps us overcome the memory limit
log.info('process {} will write {} ts (incr/cuml) tifs of '
'total {}'.format(mpiops.rank, len(process_tifs), no_ts_tifs * 2))
for i in process_tifs:
tscum_g = np.empty(shape=ifgs[0].shape, dtype=np.float32)
if i < no_ts_tifs:
for n, t in enumerate(tiles):
tscum_file = os.path.join(output_dir,
'tscuml_{}.npy'.format(n))
tscum = np.load(file=tscum_file)
md[ifc.EPOCH_DATE] = epochlist.dates[i + 1]
# sequence position; first time slice is #0
md['SEQUENCE_POSITION'] = i + 1
tscum_g[t.top_left_y:t.bottom_right_y,
t.top_left_x:t.bottom_right_x] = tscum[:, :, i]
dest = os.path.join(
params[cf.OUT_DIR],
'tscuml' + "_" + str(epochlist.dates[i + 1]) + ".tif")
md[ifc.DATA_TYPE] = ifc.CUML
shared.write_output_geotiff(md, gt, wkt, tscum_g, dest, np.nan)
else:
tsincr_g = np.empty(shape=ifgs[0].shape, dtype=np.float32)
i %= no_ts_tifs
for n, t in enumerate(tiles):
tsincr_file = os.path.join(output_dir,
'tsincr_{}.npy'.format(n))
tsincr = np.load(file=tsincr_file)
md[ifc.EPOCH_DATE] = epochlist.dates[i + 1]
# sequence position; first time slice is #0
md['SEQUENCE_POSITION'] = i + 1
tsincr_g[t.top_left_y:t.bottom_right_y,
t.top_left_x:t.bottom_right_x] = tsincr[:, :, i]
dest = os.path.join(
params[cf.OUT_DIR],
'tsincr' + "_" + str(epochlist.dates[i + 1]) + ".tif")
md[ifc.DATA_TYPE] = ifc.INCR
shared.write_output_geotiff(md, gt, wkt, tsincr_g, dest,
np.nan)
log.info('process {} finished writing {} ts (incr/cuml) tifs of '
'total {}'.format(mpiops.rank, len(process_tifs), no_ts_tifs * 2))
def test_transform_params():
params = {config.IFG_LKSX: 3, config.IFG_LKSY: 2, config.IFG_CROP_OPT: 1}
assert cf.transform_params(params) == (3, 2, 1)
