def setup_method(self):
self.params = Configuration(common.TEST_CONF_GAMMA).__dict__
self.tmp_dir = tempfile.mkdtemp()
self.params[cf.OUT_DIR] = self.tmp_dir
self.params[cf.REF_EST_METHOD] = 1
self.params[cf.PARALLEL] = False
self.params[cf.TMPDIR] = self.tmp_dir
common.copytree(common.SML_TEST_TIF, self.tmp_dir)
self.small_tifs = glob.glob(os.path.join(self.tmp_dir, "*.tif"))
for s in self.small_tifs:
os.chmod(s, 0o644)
self.ifgs = common.small_data_setup(self.tmp_dir, is_dir=True)
self.params[cf.INTERFEROGRAM_FILES] = [
MultiplePaths(p, self.params) for p in self.small_tifs
]
for p in self.params[cf.INTERFEROGRAM_FILES]:
p.sampled_path = p.converted_path
p.tmp_sampled_path = p.sampled_path
for ifg in self.ifgs:
ifg.close()
self.params[cf.REFX], self.params[cf.REFY] = -1, -1
self.params[cf.REFNX], self.params[cf.REFNY] = 10, 10
self.params[cf.REF_CHIP_SIZE], self.params[cf.REF_MIN_FRAC] = 21, 0.5
self.params['rows'], self.params['cols'] = 3, 2
self.params[cf.REF_PIXEL_FILE] = Configuration.ref_pixel_path(
self.params)
correct._update_params_with_tiles(self.params)
correct.ref_pixel_calc_wrapper(self.params)
def setup_class(cls):
cls.conf = common.MEXICO_CROPA_CONF
cls.params = Configuration(cls.conf).__dict__
prepifg.main(cls.params)
cls.params = Configuration(cls.conf).__dict__
multi_paths = cls.params[C.INTERFEROGRAM_FILES]
cls.ifg_paths = [p.tmp_sampled_path for p in multi_paths]
def setup_class(cls):
cls.conf = TEST_CONF_GAMMA
params = Configuration(cls.conf).__dict__
conv2tif.main(params)
params = Configuration(cls.conf).__dict__
prepifg.main(params)
cls.params = params
def test_prepifg_treats_inputs_and_outputs_read_only(gamma_conf, tempdir,
coh_mask):
tdir = Path(tempdir())
params = common.manipulate_test_conf(gamma_conf, tdir)
params[C.COH_MASK] = coh_mask
output_conf = tdir.joinpath('conf.cfg')
pyrate.configuration.write_config_file(params=params,
output_conf_file=output_conf)
params = Configuration(output_conf.as_posix()).__dict__
conv2tif.main(params)
tifs = list(Path(params[C.INTERFEROGRAM_DIR]).glob('*_unw.tif'))
assert len(tifs) == 17
params = Configuration(output_conf.as_posix()).__dict__
prepifg.main(params)
cropped_ifgs = list(Path(params[C.INTERFEROGRAM_DIR]).glob('*_ifg.tif'))
cropped_cohs = list(Path(params[C.COHERENCE_DIR]).glob('*_coh.tif'))
cropped_dem = list(Path(params[C.GEOMETRY_DIR]).glob('*_dem.tif'))
if params[C.COH_FILE_LIST] is not None: # 17 + 1 dem + 17 coh files
assert len(cropped_ifgs) + len(cropped_cohs) + len(cropped_dem) == 35
else: # 17 + 1 dem
assert len(cropped_ifgs) + len(cropped_cohs) + len(cropped_dem) == 18
# check all tifs from conv2tif are still readonly
for t in tifs:
assert t.stat().st_mode == 33060
# check all prepifg outputs are readonly
for c in cropped_cohs + cropped_ifgs + cropped_dem:
assert c.stat().st_mode == 33060
def test_prepifg_largetifs_vs_python(modified_config_largetifs, gamma_conf, create_mpi_files):
print("\n\n")
print("===x==="*10)
if GITHUB_ACTIONS and np.random.randint(0, 1000) > 499: # skip 50% of tests randomly
pytest.skip("Randomly skipping as part of 50 percent")
params = create_mpi_files(gamma_conf)
sr_conf, params_p = modified_config_largetifs(gamma_conf, 1, 'parallel_conf.conf')
params_p = Configuration(sr_conf).__dict__
conv2tif.main(params_p)
params_p = Configuration(sr_conf).__dict__
prepifg.main(params_p)
params_p = Configuration(sr_conf).__dict__
# convert2tif tests, 17 interferograms
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "*_unw.tif", 17)
# if coherence masking, compare coh files were converted
if params[cf.COH_FILE_LIST] is not None:
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "*_cc.tif", 17)
# 17 ifgs + 1 dem + 17 mlooked file
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "*_coh.tif", 17)
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "*_dem.tif", 1)
# prepifg
# 17 ifgs + 1 dem
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "*_ifg.tif", 17)
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "dem.tif", 1)
print("==========================xxx===========================")
shutil.rmtree(params[cf.OBS_DIR])
shutil.rmtree(params_p[cf.OBS_DIR])
def setup_class(cls):
cls.conf = TEST_CONF_GAMMA
cls.params = Configuration(cls.conf).__dict__
conv2tif.main(cls.params)
cls.params = Configuration(cls.conf).__dict__
prepifg.main(cls.params)
cls.params = Configuration(cls.conf).__dict__
multi_paths = cls.params[cf.INTERFEROGRAM_FILES]
cls.ifg_paths = [p.tmp_sampled_path for p in multi_paths]
def setup_class(cls):
cls.conf = TEST_CONF_GAMMA
params = Configuration(cls.conf).__dict__
conv2tif.main(params)
params = Configuration(cls.conf).__dict__
prepifg.main(params)
cls.params = Configuration(cls.conf).__dict__
correct._copy_mlooked(cls.params)
correct._create_ifg_dict(cls.params)
def _create(conf):
mpi_conf, params = modified_config_short(conf, 0, 'mpi_conf.conf')
params = Configuration(mpi_conf).__dict__
conv2tif.main(params)
params = Configuration(mpi_conf).__dict__
prepifg.main(params)
return params # don't need the reamining params
def setup_method(cls):
cls.conf = TEST_CONF_GAMMA
params = Configuration(cls.conf).__dict__
conv2tif.main(params)
params = Configuration(cls.conf).__dict__
prepifg.main(params)
params = Configuration(cls.conf).__dict__
correct._copy_mlooked(params)
cls.params = params
def __merge_setup(params):
"""
Convenience function for Merge set up steps
"""
# load previously saved preread_ifgs dict
preread_ifgs_file = Configuration.preread_ifgs(params)
ifgs_dict = pickle.load(open(preread_ifgs_file, 'rb'))
ifgs = [v for v in ifgs_dict.values() if isinstance(v, shared.PrereadIfg)]
shape = ifgs[0].shape
tiles = Configuration.get_tiles(params)
return shape, tiles, ifgs_dict
def setup_method(cls):
cls.conf = TEST_CONF_GAMMA
params = Configuration(cls.conf).__dict__
conv2tif.main(params)
params = Configuration(cls.conf).__dict__
prepifg.main(params)
cls.params = Configuration(cls.conf).__dict__
correct._copy_mlooked(cls.params)
correct._create_ifg_dict(cls.params)
multi_paths = cls.params[cf.INTERFEROGRAM_FILES]
cls.ifg_paths = [p.tmp_sampled_path for p in multi_paths]
def setup_class(cls):
cls.conf = common.MEXICO_CROPA_CONF
cls.params = Configuration(cls.conf).__dict__
prepifg.main(cls.params)
cls.params = Configuration(cls.conf).__dict__
multi_paths = cls.params[C.INTERFEROGRAM_FILES]
cls.ifg_paths = [p.tmp_sampled_path for p in multi_paths]
cls.params[
C.
REFX_FOUND] = 8 # this is the pixel of the location given in the pyrate_mexico_cropa.conf file
cls.params[
C.
REFY_FOUND] = 33 # however, the median of the whole interferogram is used for this validation
def setup_class(cls):
# testing constants2
cls.BASE_DIR = tempfile.mkdtemp()
cls.BASE_OUT_DIR = join(cls.BASE_DIR, 'out')
cls.BASE_DEM_DIR = join(cls.BASE_DIR, 'dem')
cls.BASE_DEM_FILE = join(cls.BASE_DEM_DIR, 'roipac_test_trimmed.tif')
try:
# copy source data (treat as prepifg already run)
os.makedirs(cls.BASE_OUT_DIR)
for path in glob.glob(join(common.SML_TEST_TIF, '*')):
dest = join(cls.BASE_OUT_DIR, os.path.basename(path))
shutil.copy(path, dest)
os.chmod(dest, 0o660)
config = Configuration(common.TEST_CONF_ROIPAC)
params = config.__dict__
params['correct'] = [
'orbfit', 'refphase', 'mst', 'apscorrect', 'maxvar'
]
params[C.OUT_DIR] = cls.BASE_OUT_DIR
params[C.PROCESSOR] = 0 # roipac
params[C.APS_CORRECTION] = 0
paths = glob.glob(join(cls.BASE_OUT_DIR, 'geo_*-*.tif'))
paths = sorted(paths)
params[C.PARALLEL] = False
params[C.ORBFIT_OFFSET] = True
params[C.TEMP_MLOOKED_DIR] = cls.BASE_OUT_DIR.join(
C.TEMP_MLOOKED_DIR)
params[C.INTERFEROGRAM_FILES] = [
MultiplePaths(p, params) for p in paths
]
for p in params[C.INTERFEROGRAM_FILES]: # cheat
p.sampled_path = p.converted_path
p.tmp_sampled_path = p.converted_path
params["rows"], params["cols"] = 2, 2
params[C.REF_PIXEL_FILE] = Configuration.ref_pixel_path(params)
Path(params[C.OUT_DIR]).joinpath(C.APS_ERROR_DIR).mkdir(
exist_ok=True, parents=True)
Path(params[C.OUT_DIR]).joinpath(C.MST_DIR).mkdir(exist_ok=True,
parents=True)
correct.correct_ifgs(config)
if not hasattr(cls, 'ifgs'):
cls.ifgs = get_ifgs(out_dir=cls.BASE_OUT_DIR)
except:
# revert working dir & avoid paths busting other tests
os.chdir(CURRENT_DIR)
raise
def stack_calc_wrapper(params):
"""
Wrapper for stacking on a set of tiles.
"""
log.info('Calculating rate map via stacking')
if not Configuration.vcmt_path(params).exists():
raise FileNotFoundError(
"VCMT is not found on disc. Have you run the 'correct' step?")
params[C.PREREAD_IFGS] = cp.load(
open(Configuration.preread_ifgs(params), 'rb'))
params[C.VCMT] = np.load(Configuration.vcmt_path(params))
params[C.TILES] = Configuration.get_tiles(params)
tiles_split(_stacking_for_tile, params)
log.debug("Finished stacking calc!")
def timeseries_calc_wrapper(params):
"""
Wrapper for time series calculation on a set of tiles.
"""
if params[C.TIME_SERIES_METHOD] == 1:
log.info('Calculating time series using Laplacian Smoothing method')
elif params[C.TIME_SERIES_METHOD] == 2:
log.info('Calculating time series using SVD method')
if not Configuration.vcmt_path(params).exists():
raise FileNotFoundError("VCMT is not found on disc. Have you run the 'correct' step?")
params[C.PREREAD_IFGS] = cp.load(open(Configuration.preread_ifgs(params), 'rb'))
params[C.VCMT] = np.load(Configuration.vcmt_path(params))
params[C.TILES] = Configuration.get_tiles(params)
tiles_split(__calc_time_series_for_tile, params)
log.debug("Finished timeseries calc!")
def __save_ifgs_dict_with_headers_and_epochs(dest_tifs, ifgs_dict, params,
process_tifs):
tmpdir = params[cf.TMPDIR]
if not os.path.exists(tmpdir):
shared.mkdir_p(tmpdir)
preread_ifgs_file = Configuration.preread_ifgs(params)
nifgs = len(dest_tifs)
# add some extra information that's also useful later
gt, md, wkt = shared.get_geotiff_header_info(
process_tifs[0].tmp_sampled_path)
epochlist = algorithm.get_epochs(ifgs_dict)[0]
log.info('Found {} unique epochs in the {} interferogram network'.format(
len(epochlist.dates), nifgs))
ifgs_dict['epochlist'] = epochlist
ifgs_dict['gt'] = gt
ifgs_dict['md'] = md
ifgs_dict['wkt'] = wkt
# dump ifgs_dict file for later use
cp.dump(ifgs_dict, open(preread_ifgs_file, 'wb'))
for k in ['gt', 'epochlist', 'md', 'wkt']:
ifgs_dict.pop(k)
return ifgs_dict
def setup_class(cls):
cls.xs = 0.000833333
cls.ys = -cls.xs
cls.ifgs, cls.random_dir = diff_exts_ifgs()
cls.ifg_paths = [i.data_path for i in cls.ifgs]
cls.params = Configuration(common.TEST_CONF_ROIPAC).__dict__
cls.params[C.OUT_DIR] = cls.random_dir
cls.params[C.GEOMETRY_DIR] = Path(cls.random_dir).joinpath(
C.GEOMETRY_DIR)
cls.params[C.GEOMETRY_DIR].mkdir(exist_ok=True)
cls.params[C.INTERFEROGRAM_DIR] = Path(cls.random_dir).joinpath(
C.INTERFEROGRAM_DIR)
cls.params[C.INTERFEROGRAM_DIR].mkdir(exist_ok=True)
cls.headers = [
roipac.roipac_header(i.data_path, cls.params) for i in cls.ifgs
]
paths = [
"060619-061002_ifg.tif", "060619-061002_ifg.tif",
"060619-061002_ifg.tif", "060619-061002_ifg.tif",
"070326-070917_ifg.tif", "070326-070917_ifg.tif",
"070326-070917_ifg.tif", "070326-070917_ifg.tif"
]
cls.exp_files = [
join(cls.random_dir, C.INTERFEROGRAM_DIR, p) for p in paths
]
def test_single_workflow(gamma_conf):
check_call(f"mpirun -n 4 pyrate workflow -f {gamma_conf}", shell=True)
params = Configuration(gamma_conf).__dict__
log_file_name = 'pyrate.log.' + 'workflow'
files = list(Path(params[cf.OUT_DIR]).glob(log_file_name + '.*'))
assert len(files) == 1
# ref pixel file generated
ref_pixel_file = params[cf.REF_PIXEL_FILE]
assert Path(ref_pixel_file).exists()
ref_pixel = np.load(ref_pixel_file)
np.testing.assert_array_equal(ref_pixel, [38, 58])
# assert orbfit exists on disc
from pyrate.core import shared
looked_files = [p.sampled_path for p in params[cf.INTERFEROGRAM_FILES]]
ifgs = [shared.Ifg(ifg) for ifg in looked_files]
orbfits_on_disc = [
Path(params[cf.OUT_DIR], cf.ORB_ERROR_DIR,
Path(ifg.data_path).stem + '_orbfit.npy') for ifg in ifgs
]
assert all(orbfits_on_disc)
shutil.rmtree(params[cf.OUT_DIR])
def series_ifgs(gamma_conf):
print('======================setup series==========================')
tdir = Path(tempfile.mkdtemp())
params_s = manipulate_test_conf(gamma_conf, tdir)
params_s[cf.PARALLEL] = 0
output_conf_file = 'conf.conf'
output_conf = tdir.joinpath(output_conf_file)
pyrate.configuration.write_config_file(params=params_s,
output_conf_file=output_conf)
params_s = Configuration(output_conf).__dict__
conv2tif.main(params_s)
prepifg.main(params_s)
serial_ifgs = list(Path(tdir).joinpath('out').glob(ifg_glob_suffix))
coh_files = list(Path(tdir).joinpath('out').glob(coh_glob_suffix))
s_ifgs = small_data_setup(datafiles=serial_ifgs + coh_files)
yield s_ifgs
print('======================teardown series==========================')
shutil.rmtree(params_s[cf.OBS_DIR], ignore_errors=True)
def series_ifgs(gamma_conf):
print('======================setup series==========================')
tdir = Path(tempfile.mkdtemp())
params_s = manipulate_test_conf(gamma_conf, tdir)
params_s[C.PARALLEL] = 0
output_conf_file = 'conf.conf'
output_conf = tdir.joinpath(output_conf_file)
pyrate.configuration.write_config_file(params=params_s,
output_conf_file=output_conf)
params_s = Configuration(output_conf).__dict__
conv2tif.main(params_s)
prepifg.main(params_s)
serial_ifgs = list(
Path(params_s[C.INTERFEROGRAM_DIR]).glob(ifg_glob_suffix))
coh_files = list(Path(params_s[C.COHERENCE_DIR]).glob(coh_glob_suffix))
s_ifgs = small_data_setup(datafiles=serial_ifgs + coh_files)
yield s_ifgs
def _calc_svd_time_series(ifg_paths, params, preread_ifgs,
tiles: List[shared.Tile]):
"""
Helper function to obtain time series for spatio-temporal filter
using SVD method
"""
# Is there other existing functions that can perform this same job?
log.info('Calculating time series via SVD method for ' 'APS correction')
# copy params temporarily
new_params = deepcopy(params)
new_params[cf.TIME_SERIES_METHOD] = 2 # use SVD method
process_tiles = mpiops.array_split(tiles)
nvels = None
for t in process_tiles:
log.debug(
'Calculating time series for tile {} during APS correction'.format(
t.index))
ifg_parts = [
shared.IfgPart(p, t, preread_ifgs, params) for p in ifg_paths
]
mst_tile = np.load(Configuration.mst_path(params, t.index))
tsincr = time_series(ifg_parts, new_params, vcmt=None, mst=mst_tile)[0]
np.save(file=os.path.join(params[cf.TMPDIR],
'tsincr_aps_{}.npy'.format(t.index)),
arr=tsincr)
nvels = tsincr.shape[2]
nvels = mpiops.comm.bcast(nvels, root=0)
mpiops.comm.barrier()
# need to assemble tsincr from all processes
tsincr_g = _assemble_tsincr(ifg_paths, params, preread_ifgs, tiles, nvels)
log.debug('Finished calculating time series for spatio-temporal filter')
return tsincr_g
def setUpClass(cls):
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['prepifg.py', common.TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
conv2tif.main(params)
prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST],
params[cf.OBS_DIR])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
# start run_pyrate copy
ifgs = common.pre_prepare_ifgs(dest_paths, params)
mst_grid = common.mst_calculation(dest_paths, params)
refx, refy = process._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.core.orbital.remove_orbital_error(ifgs, params)
ifgs = common.prepare_ifgs_without_phase(dest_paths, params)
for ifg in ifgs:
ifg.close()
_, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy)
ifgs[0].open()
r_dist = covariance.RDist(ifgs[0])()
ifgs[0].close()
maxvar = [covariance.cvd(i, params, r_dist)[0] for i in dest_paths]
for ifg in ifgs:
ifg.open()
vcmt = covariance.get_vcmt(ifgs, maxvar)
for ifg in ifgs:
ifg.close()
ifg.open()
ifg.nodata_value = 0.0
params[cf.TIME_SERIES_METHOD] = 1
params[cf.PARALLEL] = 0
# Calculate time series
cls.tsincr_0, cls.tscum_0, _ = common.calculate_time_series(
ifgs, params, vcmt, mst=mst_grid)
params[cf.PARALLEL] = 1
cls.tsincr_1, cls.tscum_1, cls.tsvel_1 = common.calculate_time_series(
ifgs, params, vcmt, mst=mst_grid)
# load the legacy data
ts_dir = os.path.join(common.SML_TEST_DIR, 'time_series')
tsincr_path = os.path.join(ts_dir, 'ts_incr_interp0_method1.csv')
ts_incr = np.genfromtxt(tsincr_path)
tscum_path = os.path.join(ts_dir, 'ts_cum_interp0_method1.csv')
ts_cum = np.genfromtxt(tscum_path)
cls.ts_incr = np.reshape(ts_incr,
newshape=cls.tsincr_0.shape,
order='F')
cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F')
def setUpClass(cls):
params = Configuration(TEST_CONF_ROIPAC).__dict__
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['prepifg.py', TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.TMPDIR] = os.path.join(cls.temp_out_dir, cf.TMPDIR)
shared.mkdir_p(params[cf.TMPDIR])
params[cf.REF_EST_METHOD] = 2
conv2tif.main(params)
prepifg.main(params)
cls.params = params
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST],
params[cf.OBS_DIR])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
ifgs = common.pre_prepare_ifgs(dest_paths, params)
refx, refy = process._ref_pixel_calc(dest_paths, params)
pyrate.core.orbital.remove_orbital_error(ifgs, params)
ifgs = prepare_ifgs_without_phase(dest_paths, params)
for ifg in ifgs:
ifg.close()
_, cls.ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy)
ifgs[0].open()
r_dist = RDist(ifgs[0])()
ifgs[0].close()
# Calculate interferogram noise
cls.maxvar = [cvd(i, params, r_dist, calc_alpha=True,
save_acg=True, write_vals=True)[0] for i in dest_paths]
cls.vcmt = get_vcmt(ifgs, cls.maxvar)
for ifg in ifgs:
ifg.close()
def test_plot_closure(mexico_cropa_params):
config = Configuration(MEXICO_CROPA_CONF)
params = config.__dict__
check_call(f"mpirun -n 3 pyrate prepifg -f {MEXICO_CROPA_CONF}",
shell=True)
correct._copy_mlooked(params)
correct.__validate_correct_steps(params)
# house keeping
correct._update_params_with_tiles(params)
correct._create_ifg_dict(params)
params[C.REFX_FOUND], params[
C.REFY_FOUND] = correct.ref_pixel_calc_wrapper(params)
# run through the correct steps in user specified sequence
for step in steps:
if step == 'phase_closure':
correct.correct_steps[step](params, config)
else:
correct.correct_steps[step](params)
closure_plot_file = Path(config.phase_closure_dir).joinpath(
f'closure_loops_iteration_1_fig_0.png')
assert closure_plot_file.exists()
shutil.rmtree(params[C.OUT_DIR], ignore_errors=True)
def setUp(self):
self.BASE_DIR = tempfile.mkdtemp()
# change to orbital error correction method 2
self.params = Configuration(common.TEST_CONF_ROIPAC).__dict__
self.params[cf.ORBITAL_FIT_METHOD] = NETWORK_METHOD
self.params[cf.ORBITAL_FIT_LOOKS_X] = 1
self.params[cf.ORBITAL_FIT_LOOKS_Y] = 1
data_paths = [
os.path.join(SML_TEST_TIF, p) for p in small_ifg_file_list()
]
self.new_data_paths = [
os.path.join(self.BASE_DIR, os.path.basename(d))
for d in data_paths
]
for d in data_paths:
d_copy = os.path.join(self.BASE_DIR, os.path.basename(d))
shutil.copy(d, d_copy)
os.chmod(d_copy, 0o660)
self.ifgs = small_data_setup(datafiles=self.new_data_paths)
self.headers = [
roipac.roipac_header(i.data_path, self.params) for i in self.ifgs
]
for i in self.ifgs:
if not i.is_open:
i.open()
if not i.nan_converted:
i.convert_to_nans()
if not i.mm_converted:
i.convert_to_mm()
i.write_modified_phase()
def _stacking_for_tile(tile, params):
"""
Wrapper for stacking calculation on a single tile
"""
preread_ifgs = params[C.PREREAD_IFGS]
vcmt = params[C.VCMT]
ifg_paths = [
ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]
]
output_dir = params[C.TMPDIR]
log.debug(f"Stacking of tile {tile.index}")
ifg_parts = [
shared.IfgPart(p, tile, preread_ifgs, params) for p in ifg_paths
]
mst_tile = np.load(Configuration.mst_path(params, tile.index))
rate, error, samples = stack_rate_array(ifg_parts, params, vcmt, mst_tile)
np.save(file=os.path.join(output_dir,
'stack_rate_{}.npy'.format(tile.index)),
arr=rate)
np.save(file=os.path.join(output_dir,
'stack_error_{}.npy'.format(tile.index)),
arr=error)
np.save(file=os.path.join(output_dir,
'stack_samples_{}.npy'.format(tile.index)),
arr=samples)
def test_ref_pixel_multiple_runs_reuse_from_disc(self, ref_pixel):
params = self.params
params[C.REFX], params[C.REFY] = ref_pixel
params[C.REF_PIXEL_FILE] = Configuration.ref_pixel_path(params)
ref_pixel_calc_wrapper(params)
ref_pixel_file = self.params[C.REF_PIXEL_FILE]
time_written = os.stat(ref_pixel_file).st_mtime
assert self.params[C.REFX_FOUND] == 38
assert self.params[C.REFY_FOUND] == 58
# run again
ref_pixel_calc_wrapper(self.params)
ref_pixel_file = self.params[C.REF_PIXEL_FILE]
time_written_1 = os.stat(ref_pixel_file).st_mtime
assert self.params[C.REFX_FOUND] == 38
assert self.params[C.REFY_FOUND] == 58
# run a third time
ref_pixel_calc_wrapper(self.params)
ref_pixel_file = self.params[C.REF_PIXEL_FILE]
time_written_2 = os.stat(ref_pixel_file).st_mtime
assert time_written == time_written_2 == time_written_1
assert self.params[C.REFX], self.params[C.REFY] == ref_pixel
assert self.params[C.REFX_FOUND] == 38
assert self.params[C.REFY_FOUND] == 58
def setup_class(cls):
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
params[cf.TEMP_MLOOKED_DIR] = os.path.join(params[cf.OUT_DIR], cf.TEMP_MLOOKED_DIR)
conv2tif.main(params)
prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
xlks, _, crop = cf.transform_params(params)
dest_paths, headers = common.repair_params_for_correct_tests(params[cf.OUT_DIR], params)
correct._copy_mlooked(params)
copied_dest_paths = [os.path.join(params[cf.TEMP_MLOOKED_DIR], os.path.basename(d)) for d in dest_paths]
del dest_paths
# start run_pyrate copy
ifgs = pre_prepare_ifgs(copied_dest_paths, params)
mst_grid = tests.common.mst_calculation(copied_dest_paths, params)
refx, refy = pyrate.core.refpixel.ref_pixel_calc_wrapper(params)
params[cf.REFX] = refx
params[cf.REFY] = refy
params[cf.ORBFIT_OFFSET] = True
# Estimate and remove orbit errors
pyrate.core.orbital.remove_orbital_error(ifgs, params)
ifgs = prepare_ifgs_without_phase(copied_dest_paths, params)
for ifg in ifgs:
ifg.close()
correct._update_params_with_tiles(params)
_, ifgs = pyrate.core.ref_phs_est.ref_phase_est_wrapper(params)
ifgs[0].open()
r_dist = vcm_module.RDist(ifgs[0])()
ifgs[0].close()
maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in copied_dest_paths]
for ifg in ifgs:
ifg.open()
vcmt = vcm_module.get_vcmt(ifgs, maxvar)
for ifg in ifgs:
ifg.close()
ifg.open()
# Calculate stacked rate map
params[cf.PARALLEL] = 1
cls.rate, cls.error, cls.samples = tests.common.calculate_stack_rate(ifgs, params, vcmt, mst_mat=mst_grid)
# Calculate stacked rate map
params[cf.PARALLEL] = 0
cls.rate_s, cls.error_s, cls.samples_s = tests.common.calculate_stack_rate(ifgs, params, vcmt, mst_mat=mst_grid)
stackrate_dir = os.path.join(SML_TEST_DIR, 'stackrate')
cls.rate_container = np.genfromtxt(os.path.join(stackrate_dir, 'stackmap.csv'), delimiter=',')
cls.error_container = np.genfromtxt(os.path.join(stackrate_dir, 'errormap.csv'), delimiter=',')
cls.samples_container = np.genfromtxt(os.path.join(stackrate_dir, 'coh_sta.csv'), delimiter=',')
for ifg in ifgs:
ifg.close()
cls.params = params
def common_check(self, ele, inc):
import glob
from pyrate.configuration import Configuration
assert os.path.exists(self.conf_file)
params = Configuration(self.conf_file).__dict__
conv2tif.main(params)
sys.argv = ['dummy', self.conf_file]
prepifg.main(params)
# test 17 geotiffs created
geotifs = glob.glob(os.path.join(params[cf.OUT_DIR], '*_unw_ifg.tif'))
self.assertEqual(17, len(geotifs))
# test dem geotiff created
demtif = glob.glob(os.path.join(params[cf.OUT_DIR], '*_dem.tif'))
self.assertEqual(1, len(demtif))
# elevation/incidence file
# not computing anymore
# ele = glob.glob(os.path.join(params[cf.OBS_DIR],
# '*utm_{ele}.tif'.format(ele=ele)))[0]
# self.assertTrue(os.path.exists(ele))
# mlooked tifs
mlooked_tifs = glob.glob(os.path.join(self.base_dir, '*_1cr.tif'))
# 19 including 17 ifgs, 1 dem and one incidence
self.assertEqual(18, len(mlooked_tifs))
inc = glob.glob(
os.path.join(self.base_dir, '*utm_{inc}.tif'.format(inc=inc)))
self.assertEqual(0, len(inc))
def maxvar_vcm_calc_wrapper(params):
"""
MPI wrapper for maxvar and vcmt computation
"""
preread_ifgs = params[cf.PREREAD_IFGS]
ifg_paths = [ifg_path.tmp_sampled_path for ifg_path in params[cf.INTERFEROGRAM_FILES]]
log.info('Calculating the temporal variance-covariance matrix')
def _get_r_dist(ifg_path):
"""
Get RDIst class object
"""
ifg = Ifg(ifg_path)
ifg.open()
r_dist = RDist(ifg)()
ifg.close()
return r_dist
r_dist = mpiops.run_once(_get_r_dist, ifg_paths[0])
prcs_ifgs = mpiops.array_split(list(enumerate(ifg_paths)))
process_maxvar = {}
for n, i in prcs_ifgs:
log.debug(f'Calculating maxvar for {n} of process ifgs {len(prcs_ifgs)} of total {len(ifg_paths)}')
process_maxvar[int(n)] = cvd(i, params, r_dist, calc_alpha=True, write_vals=True, save_acg=True)[0]
maxvar_d = shared.join_dicts(mpiops.comm.allgather(process_maxvar))
maxvar = [v[1] for v in sorted(maxvar_d.items(), key=lambda s: s[0])]
vcmt = mpiops.run_once(get_vcmt, preread_ifgs, maxvar)
log.debug("Finished maxvar and vcm calc!")
params[cf.MAXVAR], params[cf.VCMT] = maxvar, vcmt
np.save(Configuration.vcmt_path(params), arr=vcmt)
return maxvar, vcmt
