def setUpClass(cls):
params = Configuration(TEST_CONF_ROIPAC).__dict__
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['prepifg.py', TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.TMPDIR] = os.path.join(cls.temp_out_dir, cf.TMPDIR)
shared.mkdir_p(params[cf.TMPDIR])
params[cf.REF_EST_METHOD] = 2
conv2tif.main(params)
prepifg.main(params)
cls.params = params
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST],
params[cf.OBS_DIR])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
ifgs = common.pre_prepare_ifgs(dest_paths, params)
refx, refy = process._ref_pixel_calc(dest_paths, params)
pyrate.core.orbital.remove_orbital_error(ifgs, params)
ifgs = prepare_ifgs_without_phase(dest_paths, params)
for ifg in ifgs:
ifg.close()
_, cls.ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy)
ifgs[0].open()
r_dist = RDist(ifgs[0])()
ifgs[0].close()
# Calculate interferogram noise
cls.maxvar = [cvd(i, params, r_dist, calc_alpha=True,
save_acg=True, write_vals=True)[0] for i in dest_paths]
cls.vcmt = get_vcmt(ifgs, cls.maxvar)
for ifg in ifgs:
ifg.close()
def setUpClass(cls):
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['prepifg.py', common.TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
conv2tif.main(params)
prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST],
params[cf.OBS_DIR])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
# start run_pyrate copy
ifgs = common.pre_prepare_ifgs(dest_paths, params)
mst_grid = common.mst_calculation(dest_paths, params)
refx, refy = process._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.core.orbital.remove_orbital_error(ifgs, params)
ifgs = common.prepare_ifgs_without_phase(dest_paths, params)
for ifg in ifgs:
ifg.close()
_, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy)
ifgs[0].open()
r_dist = covariance.RDist(ifgs[0])()
ifgs[0].close()
maxvar = [covariance.cvd(i, params, r_dist)[0] for i in dest_paths]
for ifg in ifgs:
ifg.open()
vcmt = covariance.get_vcmt(ifgs, maxvar)
for ifg in ifgs:
ifg.close()
ifg.open()
ifg.nodata_value = 0.0
params[cf.TIME_SERIES_METHOD] = 1
params[cf.PARALLEL] = 0
# Calculate time series
cls.tsincr_0, cls.tscum_0, _ = common.calculate_time_series(
ifgs, params, vcmt, mst=mst_grid)
params[cf.PARALLEL] = 1
cls.tsincr_1, cls.tscum_1, cls.tsvel_1 = common.calculate_time_series(
ifgs, params, vcmt, mst=mst_grid)
# load the legacy data
ts_dir = os.path.join(common.SML_TEST_DIR, 'time_series')
tsincr_path = os.path.join(ts_dir, 'ts_incr_interp0_method1.csv')
ts_incr = np.genfromtxt(tsincr_path)
tscum_path = os.path.join(ts_dir, 'ts_cum_interp0_method1.csv')
ts_cum = np.genfromtxt(tscum_path)
cls.ts_incr = np.reshape(ts_incr,
newshape=cls.tsincr_0.shape,
order='F')
cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F')
def setup_class(cls):
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
params[cf.TEMP_MLOOKED_DIR] = os.path.join(params[cf.OUT_DIR], cf.TEMP_MLOOKED_DIR)
conv2tif.main(params)
prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
xlks, _, crop = cf.transform_params(params)
dest_paths, headers = common.repair_params_for_correct_tests(params[cf.OUT_DIR], params)
correct._copy_mlooked(params)
copied_dest_paths = [os.path.join(params[cf.TEMP_MLOOKED_DIR], os.path.basename(d)) for d in dest_paths]
del dest_paths
# start run_pyrate copy
ifgs = pre_prepare_ifgs(copied_dest_paths, params)
mst_grid = tests.common.mst_calculation(copied_dest_paths, params)
refx, refy = pyrate.core.refpixel.ref_pixel_calc_wrapper(params)
params[cf.REFX] = refx
params[cf.REFY] = refy
params[cf.ORBFIT_OFFSET] = True
# Estimate and remove orbit errors
pyrate.core.orbital.remove_orbital_error(ifgs, params)
ifgs = prepare_ifgs_without_phase(copied_dest_paths, params)
for ifg in ifgs:
ifg.close()
correct._update_params_with_tiles(params)
_, ifgs = pyrate.core.ref_phs_est.ref_phase_est_wrapper(params)
ifgs[0].open()
r_dist = vcm_module.RDist(ifgs[0])()
ifgs[0].close()
maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in copied_dest_paths]
for ifg in ifgs:
ifg.open()
vcmt = vcm_module.get_vcmt(ifgs, maxvar)
for ifg in ifgs:
ifg.close()
ifg.open()
# Calculate stacked rate map
params[cf.PARALLEL] = 1
cls.rate, cls.error, cls.samples = tests.common.calculate_stack_rate(ifgs, params, vcmt, mst_mat=mst_grid)
# Calculate stacked rate map
params[cf.PARALLEL] = 0
cls.rate_s, cls.error_s, cls.samples_s = tests.common.calculate_stack_rate(ifgs, params, vcmt, mst_mat=mst_grid)
stackrate_dir = os.path.join(SML_TEST_DIR, 'stackrate')
cls.rate_container = np.genfromtxt(os.path.join(stackrate_dir, 'stackmap.csv'), delimiter=',')
cls.error_container = np.genfromtxt(os.path.join(stackrate_dir, 'errormap.csv'), delimiter=',')
cls.samples_container = np.genfromtxt(os.path.join(stackrate_dir, 'coh_sta.csv'), delimiter=',')
for ifg in ifgs:
ifg.close()
cls.params = params
def _merge_setup(rows, cols, params):
"""
Convenience function for Merge set up steps
"""
# setup paths
xlks, _, crop = cf.transform_params(params)
base_unw_paths = []
for p in Path(params[cf.OUT_DIR]).rglob("*rlks_*cr.tif"):
if "dem" not in str(p):
base_unw_paths.append(str(p))
if "tif" in base_unw_paths[0].split(".")[1]:
dest_tifs = base_unw_paths # cf.get_dest_paths(base_unw_paths, crop, params, xlks)
for i, dest_tif in enumerate(dest_tifs):
dest_tifs[i] = dest_tif.replace("_tif", "")
else:
dest_tifs = base_unw_paths # cf.get_dest_paths(base_unw_paths, crop, params, xlks)
# load previously saved preread_ifgs dict
preread_ifgs_file = join(params[cf.TMPDIR], 'preread_ifgs.pk')
ifgs_dict = pickle.load(open(preread_ifgs_file, 'rb'))
ifgs = [v for v in ifgs_dict.values() if isinstance(v, shared.PrereadIfg)]
shape = ifgs[0].shape
tiles = shared.get_tiles(dest_tifs[0], rows, cols)
return shape, tiles, ifgs_dict
def _merge_linrate(rows, cols, params):
"""
Merge linear rate outputs
"""
# pylint: disable=expression-not-assigned
# setup paths
xlks, _, crop = cf.transform_params(params)
base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST],
params[cf.OBS_DIR])
if "tif" in base_unw_paths[0].split(".")[1]:
dest_tifs = cf.get_dest_paths(base_unw_paths, crop, params, xlks)
for i, dest_tif in enumerate(dest_tifs):
dest_tifs[i] = dest_tif.replace("_tif", "")
else:
dest_tifs = cf.get_dest_paths(base_unw_paths, crop, params, xlks)
# load previously saved prepread_ifgs dict
preread_ifgs_file = join(params[cf.TMPDIR], 'preread_ifgs.pk')
ifgs = cp.load(open(preread_ifgs_file, 'rb'))
tiles = shared.get_tiles(dest_tifs[0], rows, cols)
# linrate aggregation
if mpiops.size >= 3:
[
_save_linrate(ifgs, params, tiles, out_type=t)
for i, t in enumerate(['linrate', 'linerror', 'linsamples'])
if i == mpiops.rank
]
else:
if mpiops.rank == MASTER_PROCESS:
[
_save_linrate(ifgs, params, tiles, out_type=t)
for t in ['linrate', 'linerror', 'linsamples']
]
def main(params):
"""
Main workflow function for preparing interferograms for PyRate.
:param dict params: Parameters dictionary read in from the config file
"""
# TODO: looks like ifg_paths are ordered according to ifg list
# This probably won't be a problem because input list won't be reordered
# and the original gamma generated list is ordered) this may not affect
# the important pyrate stuff anyway, but might affect gen_thumbs.py.
# Going to assume ifg_paths is ordered correcly
# pylint: disable=too-many-branches
shared.mpi_vs_multiprocess_logging("prepifg", params)
ifg_paths = params[cf.INTERFEROGRAM_FILES]
if params[cf.DEM_FILE] is not None: # optional DEM conversion
ifg_paths.append(params[cf.DEM_FILE_PATH])
if params[cf.COH_MASK]:
ifg_paths.extend(params[cf.COHERENCE_FILE_PATHS])
shared.mkdir_p(params[cf.OUT_DIR]) # create output dir
user_exts = (params[cf.IFG_XFIRST], params[cf.IFG_YFIRST], params[cf.IFG_XLAST], params[cf.IFG_YLAST])
xlooks, ylooks, crop = cf.transform_params(params)
ifgs = [prepifg_helper.dem_or_ifg(p.converted_path) for p in ifg_paths]
exts = prepifg_helper.get_analysis_extent(crop, ifgs, xlooks, ylooks, user_exts=user_exts)
process_ifgs_paths = np.array_split(ifg_paths, mpiops.size)[mpiops.rank]
do_prepifg(process_ifgs_paths, exts, params)
mpiops.comm.barrier()
log.info("Finished prepifg")
def _prepifg_multiprocessing(m_path: MultiplePaths, exts: Tuple[float, float, float, float], params: dict):
"""
Multiprocessing wrapper for prepifg
"""
xlooks, ylooks, crop = cf.transform_params(params)
thresh = params[cf.NO_DATA_AVERAGING_THRESHOLD]
header = find_header(m_path, params)
header[ifc.INPUT_TYPE] = m_path.input_type
# If we're performing coherence masking, find the coherence file for this IFG.
if params[cf.COH_MASK] and shared._is_interferogram(header):
coherence_path = cf.coherence_paths_for(m_path.converted_path, params, tif=True)
coherence_thresh = params[cf.COH_THRESH]
else:
coherence_path = None
coherence_thresh = None
if params[cf.LARGE_TIFS]:
op = output_tiff_filename(m_path.converted_path, params[cf.OUT_DIR])
looks_path = cf.mlooked_path(op, ylooks, crop)
return m_path.converted_path, coherence_path, looks_path
else:
prepifg_helper.prepare_ifg(m_path.converted_path, xlooks, ylooks, exts, thresh, crop,
out_path=params[cf.OUT_DIR], header=header, coherence_path=coherence_path,
coherence_thresh=coherence_thresh)
def test_vcm_legacy_vs_mpi(mpisync, tempdir, roipac_or_gamma_conf):
params = configuration.Configuration(roipac_or_gamma_conf).__dict__
LEGACY_VCM_DIR = os.path.join(SML_TEST_DIR, 'vcm')
legacy_vcm = np.genfromtxt(os.path.join(LEGACY_VCM_DIR, 'vcmt.csv'),
delimiter=',')
tmpdir = Path(mpiops.run_once(tempdir))
mpiops.run_once(common.copytree, params[cf.OBS_DIR], tmpdir)
params[cf.OUT_DIR] = tmpdir.joinpath('out')
params[cf.PARALLEL] = False
xlks, ylks, crop = cf.transform_params(params)
base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST],
params[cf.OBS_DIR])
# dest_paths are tifs that have been geotif converted and multilooked
dest_paths = cf.get_dest_paths(base_unw_paths, crop, params, xlks)
# run conv2tif and prepifg, create the dest_paths files
conv2tif.main(params)
prepifg.main(params)
tiles = pyrate.core.shared.get_tiles(dest_paths[0], rows=1, cols=1)
preread_ifgs = process._create_ifg_dict(dest_paths,
params=params,
tiles=tiles)
refpx, refpy = process._ref_pixel_calc(dest_paths, params)
process._orb_fit_calc(dest_paths, params)
process._ref_phase_estimation(dest_paths, params, refpx, refpy)
maxvar, vcmt = process._maxvar_vcm_calc(dest_paths, params, preread_ifgs)
np.testing.assert_array_almost_equal(maxvar, legacy_maxvar, decimal=4)
np.testing.assert_array_almost_equal(legacy_vcm, vcmt, decimal=3)
mpiops.run_once(shutil.rmtree, tmpdir)
def setUpClass(cls):
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
shutil.rmtree(params[cf.OUT_DIR])
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['prepifg.py', common.TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.TMPDIR] = cls.temp_out_dir
conv2tif.main(params)
prepifg.main(params)
params[cf.REF_EST_METHOD] = 1
params[cf.PARALLEL] = True
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = [
c.unwrapped_path for c in params[cf.INTERFEROGRAM_FILES]
]
headers = [roipac.roipac_header(i, params) for i in base_ifg_paths]
dest_paths = [
Path(cls.temp_out_dir).joinpath(Path(
c.sampled_path).name).as_posix()
for c in params[cf.INTERFEROGRAM_FILES][:-2]
]
# start run_pyrate copy
ifgs = common.pre_prepare_ifgs(dest_paths, params)
mst_grid = common.mst_calculation(dest_paths, params)
# Estimate reference pixel location
refx, refy = process._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.core.orbital.remove_orbital_error(ifgs, params, headers)
for i in ifgs:
i.close()
ifgs = common.pre_prepare_ifgs(dest_paths, params)
for i in ifgs:
i.close()
cls.ref_phs, cls.ifgs = process._ref_phase_estimation(
dest_paths, params, refx, refy)
def do_prepifg(gtiff_paths: List[str], params: dict) -> None:
"""
Prepare interferograms by applying multilooking/cropping operations.
:param list gtiff_paths: List of full-res geotiffs
:param dict params: Parameters dictionary corresponding to config file
"""
# pylint: disable=expression-not-assigned
parallel = params[cf.PARALLEL]
if mpiops.size > 1:
parallel = False
for f in gtiff_paths:
if not os.path.isfile(f):
raise FileNotFoundError("Can not find geotiff: " + str(f) + ". Ensure you have converted your "
"interferograms to geotiffs.")
ifgs = [prepifg_helper.dem_or_ifg(p) for p in gtiff_paths]
xlooks, ylooks, crop = cf.transform_params(params)
user_exts = (params[cf.IFG_XFIRST], params[cf.IFG_YFIRST], params[cf.IFG_XLAST], params[cf.IFG_YLAST])
exts = prepifg_helper.get_analysis_extent(crop, ifgs, xlooks, ylooks, user_exts=user_exts)
thresh = params[cf.NO_DATA_AVERAGING_THRESHOLD]
if params[cf.LARGE_TIFS]:
log.info("Using gdal system calls to process prepifg")
ifg = ifgs[0]
res_str = [xlooks * ifg.x_step, ylooks * ifg.y_step]
res_str = ' '.join([str(e) for e in res_str])
if parallel:
Parallel(n_jobs=params[cf.PROCESSES], verbose=50)(
delayed(__prepifg_system)(
crop, exts, gtiff_path, params, res_str, thresh, xlooks, ylooks) for gtiff_path in gtiff_paths
)
else:
for gtiff_path in gtiff_paths:
__prepifg_system(crop, exts, gtiff_path, params, res_str, thresh, xlooks, ylooks)
else:
if parallel:
Parallel(n_jobs=params[cf.PROCESSES], verbose=50)(
delayed(_prepifg_multiprocessing)(p, xlooks, ylooks, exts, thresh, crop, params) for p in gtiff_paths
)
else:
for gtiff_path in gtiff_paths:
_prepifg_multiprocessing(gtiff_path, xlooks, ylooks, exts, thresh, crop, params)
def _merge_stack(rows, cols, params):
"""
Merge stacking outputs
"""
# setup paths
xlks, _, crop = cf.transform_params(params)
base_unw_paths = []
for p in Path(params[OUT_DIR]).rglob("*rlks_*cr.tif"):
if "dem" not in str(p):
base_unw_paths.append(str(p))
if not base_unw_paths:
raise ConfigException(
f"No rlooked files available in {params[OBS_DIR]} or {params[OBS_DIR]}"
)
if "tif" in base_unw_paths[0].split(".")[1]:
dest_tifs = base_unw_paths # cf.get_dest_paths(base_unw_paths, crop, params, xlks)
for i, dest_tif in enumerate(dest_tifs):
dest_tifs[i] = dest_tif.replace("_tif", "")
else:
dest_tifs = base_unw_paths # cf.get_dest_paths(base_unw_paths, crop, params, xlks)
# load previously saved prepread_ifgs dict
preread_ifgs_file = join(params[cf.TMPDIR], 'preread_ifgs.pk')
ifgs = cp.load(open(preread_ifgs_file, 'rb'))
tiles = shared.get_tiles(dest_tifs[0], rows, cols)
# stacking aggregation
if mpiops.size >= 3:
[
_save_stack(ifgs, params, tiles, out_type=t) for i, t in enumerate(
['stack_rate', 'stack_error', 'stack_samples'])
if i == mpiops.rank
]
else:
if mpiops.rank == MASTER_PROCESS:
[
_save_stack(ifgs, params, tiles, out_type=t)
for t in ['stack_rate', 'stack_error', 'stack_samples']
]
mpiops.comm.barrier()
def test_vcm_legacy_vs_mpi(mpisync, tempdir, get_config):
from tests.common import SML_TEST_DIR, TEST_CONF_ROIPAC
params_dict = get_config(TEST_CONF_ROIPAC)
LEGACY_VCM_DIR = os.path.join(SML_TEST_DIR, 'vcm')
legacy_vcm = np.genfromtxt(os.path.join(LEGACY_VCM_DIR, 'vcmt.csv'),
delimiter=',')
if mpiops.rank == 0:
outdir = tempdir()
else:
outdir = None
outdir = mpiops.comm.bcast(outdir, root=0)
params_dict[cf.OUT_DIR] = outdir
params_dict[cf.PARALLEL] = False
xlks, ylks, crop = cf.transform_params(params_dict)
base_unw_paths = cf.original_ifg_paths(params_dict[cf.IFG_FILE_LIST],
params_dict[cf.OBS_DIR])
# dest_paths are tifs that have been geotif converted and multilooked
dest_paths = cf.get_dest_paths(base_unw_paths, crop, params_dict, xlks)
# run prepifg, create the dest_paths files
if mpiops.rank == 0:
conv2tif.main(params_dict)
prepifg.main(params_dict)
mpiops.comm.barrier()
tiles = pyrate.core.shared.get_tiles(dest_paths[0], rows=1, cols=1)
preread_ifgs = process._create_ifg_dict(dest_paths,
params=params_dict,
tiles=tiles)
refpx, refpy = process._ref_pixel_calc(dest_paths, params_dict)
process._orb_fit_calc(dest_paths, params_dict)
process._ref_phase_estimation(dest_paths, params_dict, refpx, refpy)
maxvar, vcmt = process._maxvar_vcm_calc(dest_paths, params_dict,
preread_ifgs)
np.testing.assert_array_almost_equal(maxvar, legacy_maxvar, decimal=4)
np.testing.assert_array_almost_equal(legacy_vcm, vcmt, decimal=3)
if mpiops.rank == 0:
shutil.rmtree(outdir)
common.remove_tifs(params_dict[cf.OBS_DIR])
def setUpClass(cls):
params = cf.get_config_params(common.TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['prepifg.py', common.TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.TMPDIR] = cls.temp_out_dir
conv2tif.main(params)
prepifg.main(params)
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.REF_EST_METHOD] = 2
params[cf.PARALLEL] = True
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST],
params[cf.OBS_DIR])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
# start run_pyrate copy
ifgs = common.pre_prepare_ifgs(dest_paths, params)
# Estimate reference pixel location
refx, refy = process._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.core.orbital.remove_orbital_error(ifgs, params)
for i in ifgs:
i.close()
ifgs = common.pre_prepare_ifgs(dest_paths, params)
for i in ifgs:
i.close()
cls.ref_phs, cls.ifgs = process._ref_phase_estimation(
dest_paths, params, refx, refy)
def _prepifg_multiprocessing(m_path: MultiplePaths, exts: Tuple[float, float,
float, float],
params: dict):
"""
Multiprocessing wrapper for prepifg
"""
thresh = params[cf.NO_DATA_AVERAGING_THRESHOLD]
hdr = find_header(m_path, params)
hdr[ifc.INPUT_TYPE] = m_path.input_type
xlooks, ylooks, crop = cf.transform_params(params)
hdr[ifc.IFG_LKSX] = xlooks
hdr[ifc.IFG_LKSY] = ylooks
hdr[ifc.IFG_CROP] = crop
# If we're performing coherence masking, find the coherence file for this IFG.
if params[cf.COH_MASK] and shared._is_interferogram(hdr):
coherence_path = cf.coherence_paths_for(m_path.converted_path,
params,
tif=True)
coherence_thresh = params[cf.COH_THRESH]
else:
coherence_path = None
coherence_thresh = None
if params[cf.LARGE_TIFS]:
return m_path.converted_path, coherence_path, m_path.sampled_path
else:
prepifg_helper.prepare_ifg(m_path.converted_path,
xlooks,
ylooks,
exts,
thresh,
crop,
out_path=m_path.sampled_path,
header=hdr,
coherence_path=coherence_path,
coherence_thresh=coherence_thresh)
Path(m_path.sampled_path).chmod(0o444) # readonly output
def do_prepifg(gtiff_paths, params):
"""
Prepare interferograms by applying multilooking/cropping operations.
:param list gtiff_paths: List of full-res geotiffs
:param dict params: Parameters dictionary corresponding to config file
"""
# pylint: disable=expression-not-assigned
log.info("Preparing interferograms by cropping/multilooking")
parallel = params[cf.PARALLEL]
for f in gtiff_paths:
if not os.path.isfile(f):
raise Exception(
"Can not find geotiff: " + str(f) +
". Ensure you have converted your interferograms to geotiffs.")
ifgs = [prepifg_helper.dem_or_ifg(p) for p in gtiff_paths]
xlooks, ylooks, crop = cf.transform_params(params)
user_exts = (params[cf.IFG_XFIRST], params[cf.IFG_YFIRST],
params[cf.IFG_XLAST], params[cf.IFG_YLAST])
exts = prepifg_helper.get_analysis_extent(crop,
ifgs,
xlooks,
ylooks,
user_exts=user_exts)
log.debug("Extents (xmin, ymin, xmax, ymax): " + str(exts))
thresh = params[cf.NO_DATA_AVERAGING_THRESHOLD]
if parallel:
Parallel(n_jobs=params[cf.PROCESSES],
verbose=50)(delayed(_prepifg_multiprocessing)(
p, xlooks, ylooks, exts, thresh, crop, params)
for p in gtiff_paths)
else:
[
_prepifg_multiprocessing(p, xlooks, ylooks, exts, thresh, crop,
params) for p in gtiff_paths
]
def setUpClass(cls):
cls.tif_dir = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_GAMMA
# change the required params
cls.params = Configuration(cls.test_conf).__dict__
cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA
cls.params[cf.PROCESSOR] = 1 # gamma
file_list = list(
cf.parse_namelist(os.path.join(common.SML_TEST_GAMMA, 'ifms_17')))
fd, cls.params[cf.IFG_FILE_LIST] = tempfile.mkstemp(suffix='.conf',
dir=cls.tif_dir)
os.close(fd)
# write a short filelist with only 3 gamma unws
with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp:
for f in file_list[:3]:
fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n')
cls.params[cf.OUT_DIR] = cls.tif_dir
cls.params[cf.PARALLEL] = 0
cls.params[cf.REF_EST_METHOD] = 1
cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
# base_unw_paths need to be geotiffed and multilooked by run_prepifg
cls.base_unw_paths = cf.original_ifg_paths(
cls.params[cf.IFG_FILE_LIST], cls.params[cf.OBS_DIR])
cls.base_unw_paths.append(common.SML_TEST_DEM_GAMMA)
xlks, ylks, crop = cf.transform_params(cls.params)
# dest_paths are tifs that have been geotif converted and multilooked
conv2tif.main(cls.params)
prepifg.main(cls.params)
# run_prepifg.gamma_prepifg(cls.base_unw_paths, cls.params)
cls.base_unw_paths.pop() # removed dem as we don't want it in ifgs
cls.dest_paths = cf.get_dest_paths(cls.base_unw_paths, crop,
cls.params, xlks)
cls.ifgs = common.small_data_setup(datafiles=cls.dest_paths)
def setUpClass(cls):
params = Configuration(TEST_CONF_ROIPAC).__dict__
cls.temp_out_dir = tempfile.mkdtemp()
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
shared.mkdir_p(params[cf.TMPDIR])
conv2tif.main(params)
prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
xlks, _, crop = cf.transform_params(params)
base_ifg_paths = [
c.unwrapped_path for c in params[cf.INTERFEROGRAM_FILES]
]
headers = [roipac.roipac_header(i, params) for i in base_ifg_paths]
dest_paths = [
Path(cls.temp_out_dir).joinpath(Path(
c.sampled_path).name).as_posix()
for c in params[cf.INTERFEROGRAM_FILES][:-2]
]
# start run_pyrate copy
ifgs = pre_prepare_ifgs(dest_paths, params)
mst_grid = tests.common.mst_calculation(dest_paths, params)
refx, refy = process._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.core.orbital.remove_orbital_error(ifgs, params, headers)
ifgs = prepare_ifgs_without_phase(dest_paths, params)
for ifg in ifgs:
ifg.close()
_, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy)
ifgs[0].open()
r_dist = vcm_module.RDist(ifgs[0])()
ifgs[0].close()
maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in dest_paths]
for ifg in ifgs:
ifg.open()
vcmt = vcm_module.get_vcmt(ifgs, maxvar)
for ifg in ifgs:
ifg.close()
ifg.open()
# Calculate stacked rate map
params[cf.PARALLEL] = 1
cls.rate, cls.error, cls.samples = tests.common.calculate_stack_rate(
ifgs, params, vcmt, mst_mat=mst_grid)
# Calculate stacked rate map
params[cf.PARALLEL] = 0
cls.rate_s, cls.error_s, cls.samples_s = tests.common.calculate_stack_rate(
ifgs, params, vcmt, mst_mat=mst_grid)
stackrate_dir = os.path.join(SML_TEST_DIR, 'stackrate')
cls.rate_container = np.genfromtxt(os.path.join(
stackrate_dir, 'stackmap.csv'),
delimiter=',')
cls.error_container = np.genfromtxt(os.path.join(
stackrate_dir, 'errormap.csv'),
delimiter=',')
cls.samples_container = np.genfromtxt(os.path.join(
stackrate_dir, 'coh_sta.csv'),
delimiter=',')
for ifg in ifgs:
ifg.close()
def _merge_timeseries(rows, cols, params):
"""
Merge time series output
"""
xlks, _, crop = cf.transform_params(params)
base_unw_paths = []
for p in Path(params[OUT_DIR]).rglob("*rlks_*cr.tif"):
if "dem" not in str(p):
base_unw_paths.append(str(p))
if "tif" in base_unw_paths[0].split(".")[1]:
dest_tifs = base_unw_paths # cf.get_dest_paths(base_unw_paths, crop, params, xlks)
for i, dest_tif in enumerate(dest_tifs):
dest_tifs[i] = dest_tif.replace("_tif", "")
else:
dest_tifs = base_unw_paths # cf.get_dest_paths(base_unw_paths, crop, params, xlks)
output_dir = params[cf.TMPDIR]
# load previously saved prepread_ifgs dict
preread_ifgs_file = join(output_dir, 'preread_ifgs.pk')
ifgs = cp.load(open(preread_ifgs_file, 'rb'))
# metadata and projections
gt, md, wkt = ifgs['gt'], ifgs['md'], ifgs['wkt']
epochlist = ifgs['epochlist']
ifgs = [v for v in ifgs.values() if isinstance(v, PrereadIfg)]
tiles = shared.get_tiles(dest_tifs[0], rows, cols)
# load the first tsincr file to determine the number of time series tifs
tsincr_file = os.path.join(output_dir, 'tsincr_0.npy')
tsincr = np.load(file=tsincr_file)
# pylint: disable=no-member
no_ts_tifs = tsincr.shape[2]
# we create 2 x no_ts_tifs as we are splitting tsincr and tscuml
# to all processes.
process_tifs = mpiops.array_split(range(2 * no_ts_tifs))
# depending on nvelpar, this will not fit in memory
# e.g. nvelpar=100, nrows=10000, ncols=10000, 32bit floats need 40GB memory
# 32 * 100 * 10000 * 10000 / 8 bytes = 4e10 bytes = 40 GB
# the double for loop helps us overcome the memory limit
log.info('Process {} writing {} timeseries tifs of '
'total {}'.format(mpiops.rank, len(process_tifs), no_ts_tifs * 2))
for i in process_tifs:
tscum_g = np.empty(shape=ifgs[0].shape, dtype=np.float32)
if i < no_ts_tifs:
for n, t in enumerate(tiles):
_assemble_tiles(i, n, t, tscum_g, output_dir, 'tscuml')
md[ifc.EPOCH_DATE] = epochlist.dates[i + 1]
# sequence position; first time slice is #0
md['SEQUENCE_POSITION'] = i + 1
dest = os.path.join(
params[cf.OUT_DIR],
'tscuml' + "_" + str(epochlist.dates[i + 1]) + ".tif")
md[ifc.DATA_TYPE] = ifc.CUML
shared.write_output_geotiff(md, gt, wkt, tscum_g, dest, np.nan)
else:
tsincr_g = np.empty(shape=ifgs[0].shape, dtype=np.float32)
i %= no_ts_tifs
for n, t in enumerate(tiles):
_assemble_tiles(i, n, t, tsincr_g, output_dir, 'tsincr')
md[ifc.EPOCH_DATE] = epochlist.dates[i + 1]
# sequence position; first time slice is #0
md['SEQUENCE_POSITION'] = i + 1
dest = os.path.join(
params[cf.OUT_DIR],
'tsincr' + "_" + str(epochlist.dates[i + 1]) + ".tif")
md[ifc.DATA_TYPE] = ifc.INCR
shared.write_output_geotiff(md, gt, wkt, tsincr_g, dest, np.nan)
mpiops.comm.barrier()
log.debug('Process {} finished writing {} timeseries tifs of '
'total {}'.format(mpiops.rank, len(process_tifs),
no_ts_tifs * 2))
def setUpClass(cls):
params = cf.get_config_params(TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
shared.mkdir_p(params[cf.TMPDIR])
conv2tif.main(params)
prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
xlks, _, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST],
params[cf.OBS_DIR])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
print(f"base_ifg_paths={base_ifg_paths}")
print(f"dest_paths={dest_paths}")
# start run_pyrate copy
ifgs = pre_prepare_ifgs(dest_paths, params)
mst_grid = tests.common.mst_calculation(dest_paths, params)
refx, refy = process._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.core.orbital.remove_orbital_error(ifgs, params)
ifgs = prepare_ifgs_without_phase(dest_paths, params)
for ifg in ifgs:
ifg.close()
_, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy)
ifgs[0].open()
r_dist = vcm_module.RDist(ifgs[0])()
ifgs[0].close()
maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in dest_paths]
for ifg in ifgs:
ifg.open()
vcmt = vcm_module.get_vcmt(ifgs, maxvar)
for ifg in ifgs:
ifg.close()
ifg.open()
# Calculate linear rate map
params[cf.PARALLEL] = 1
cls.rate, cls.error, cls.samples = tests.common.calculate_linear_rate(
ifgs, params, vcmt, mst_mat=mst_grid)
params[cf.PARALLEL] = 2
cls.rate_2, cls.error_2, cls.samples_2 = \
tests.common.calculate_linear_rate(ifgs, params, vcmt,
mst_mat=mst_grid)
params[cf.PARALLEL] = 0
# Calculate linear rate map
cls.rate_s, cls.error_s, cls.samples_s = \
tests.common.calculate_linear_rate(ifgs, params, vcmt,
mst_mat=mst_grid)
linrate_dir = os.path.join(SML_TEST_DIR, 'linrate')
cls.rate_container = np.genfromtxt(os.path.join(
linrate_dir, 'stackmap.csv'),
delimiter=',')
cls.error_container = np.genfromtxt(os.path.join(
linrate_dir, 'errormap.csv'),
delimiter=',')
cls.samples_container = np.genfromtxt(os.path.join(
linrate_dir, 'coh_sta.csv'),
delimiter=',')
for ifg in ifgs:
ifg.close()
def setup_class(cls):
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
params[cf.TEMP_MLOOKED_DIR] = os.path.join(params[cf.OUT_DIR],
cf.TEMP_MLOOKED_DIR)
conv2tif.main(params)
prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
xlks, _, crop = cf.transform_params(params)
dest_paths, headers = common.repair_params_for_correct_tests(
params[cf.OUT_DIR], params)
correct._copy_mlooked(params)
copied_dest_paths = [
os.path.join(params[cf.TEMP_MLOOKED_DIR], os.path.basename(d))
for d in dest_paths
]
del dest_paths
# start run_pyrate copy
ifgs = common.pre_prepare_ifgs(copied_dest_paths, params)
mst_grid = common.mst_calculation(copied_dest_paths, params)
refx, refy = pyrate.core.refpixel.ref_pixel_calc_wrapper(params)
params[cf.REFX] = refx
params[cf.REFY] = refy
params[cf.ORBFIT_OFFSET] = True
# Estimate and remove orbit errors
pyrate.core.orbital.remove_orbital_error(ifgs, params)
ifgs = common.prepare_ifgs_without_phase(copied_dest_paths, params)
for ifg in ifgs:
ifg.close()
correct._update_params_with_tiles(params)
_, ifgs = pyrate.core.ref_phs_est.ref_phase_est_wrapper(params)
ifgs[0].open()
r_dist = covariance.RDist(ifgs[0])()
ifgs[0].close()
# Calculate interferogram noise
maxvar = [
covariance.cvd(i, params, r_dist)[0] for i in copied_dest_paths
]
for ifg in ifgs:
ifg.open()
vcmt = covariance.get_vcmt(ifgs, maxvar)
for ifg in ifgs:
ifg.close()
ifg.open()
ifg.nodata_value = 0.0
params[cf.TIME_SERIES_METHOD] = 2
params[cf.PARALLEL] = 1
# Calculate time series
cls.tsincr, cls.tscum, _ = common.calculate_time_series(ifgs,
params,
vcmt,
mst=mst_grid)
params[cf.PARALLEL] = 0
# Calculate time series serailly by the pixel
cls.tsincr_0, cls.tscum_0, _ = common.calculate_time_series(
ifgs, params, vcmt, mst=mst_grid)
# copy legacy data
SML_TIME_SERIES_DIR = os.path.join(common.SML_TEST_DIR, 'time_series')
tsincr_path = os.path.join(SML_TIME_SERIES_DIR,
'ts_incr_interp0_method2.csv')
ts_incr = np.genfromtxt(tsincr_path)
tscum_path = os.path.join(SML_TIME_SERIES_DIR,
'ts_cum_interp0_method2.csv')
ts_cum = np.genfromtxt(tscum_path)
cls.ts_incr = np.reshape(ts_incr,
newshape=cls.tsincr_0.shape,
order='F')
cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F')
cls.params = params
def setUpClass(cls):
rate_types = ['linrate', 'linerror', 'linsamples']
cls.tif_dir = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_GAMMA
# change the required params
params = cf.get_config_params(cls.test_conf)
params[cf.OBS_DIR] = common.SML_TEST_GAMMA
params[cf.PROCESSOR] = 1 # gamma
params[cf.IFG_FILE_LIST] = os.path.join(
common.SML_TEST_GAMMA, 'ifms_17')
params[cf.OUT_DIR] = cls.tif_dir
params[cf.PARALLEL] = 1
params[cf.APS_CORRECTION] = False
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
xlks, ylks, crop = cf.transform_params(params)
# base_unw_paths need to be geotiffed by converttogeotif
# and multilooked by run_prepifg
base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST],
params[cf.OBS_DIR])
# dest_paths are tifs that have been geotif converted and multilooked
cls.dest_paths = cf.get_dest_paths(
base_unw_paths, crop, params, xlks)
gtif_paths = conv2tif.do_geotiff(base_unw_paths, params)
prepifg.do_prepifg(gtif_paths, params)
tiles = pyrate.core.shared.get_tiles(cls.dest_paths[0], 3, 3)
ifgs = common.small_data_setup()
cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = \
process.process_ifgs(cls.dest_paths, params, 3, 3)
cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate_p, cls.error_p, cls.samples_p = [
common.reconstruct_linrate(
ifgs[0].shape, tiles, params[cf.TMPDIR], t)
for t in rate_types
]
common.remove_tifs(params[cf.OBS_DIR])
# now create the non parallel version
cls.tif_dir_s = tempfile.mkdtemp()
params[cf.PARALLEL] = 0
params[cf.OUT_DIR] = cls.tif_dir_s
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
cls.dest_paths_s = cf.get_dest_paths(
base_unw_paths, crop, params, xlks)
gtif_paths = conv2tif.do_geotiff(base_unw_paths, params)
prepifg.do_prepifg(gtif_paths, params)
cls.refpixel, cls.maxvar, cls.vcmt = \
process.process_ifgs(cls.dest_paths_s, params, 3, 3)
cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate, cls.error, cls.samples = [
common.reconstruct_linrate(
ifgs[0].shape, tiles, params[cf.TMPDIR], t)
for t in rate_types
]
def test_transform_params():
params = {config.IFG_LKSX: 3, config.IFG_LKSY: 2, config.IFG_CROP_OPT: 1}
assert cf.transform_params(params) == (3, 2, 1)
