def _orb_fit_calc(ifg_paths, params, preread_ifgs=None):
"""
MPI wrapper for orbital fit correction
"""
log.info('Calculating orbital correction')
if not params[cf.ORBITAL_FIT]:
log.info('Orbital correction not required')
return
if preread_ifgs: # don't check except for mpi tests
# perform some general error/sanity checks
log.debug('Checking Orbital error correction status')
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_ORBITAL_ERROR):
log.debug('Finished Orbital error correction')
return # return if True condition returned
if params[cf.ORBITAL_FIT_METHOD] == 1:
prcs_ifgs = mpiops.array_split(ifg_paths)
orbital.remove_orbital_error(prcs_ifgs, params, preread_ifgs)
else:
# Here we do all the multilooking in one process, but in memory
# can use multiple processes if we write data to disc during
# remove_orbital_error step
# A performance comparison should be made for saving multilooked
# files on disc vs in memory single process multilooking
if mpiops.rank == MASTER_PROCESS:
orbital.remove_orbital_error(ifg_paths, params, preread_ifgs)
mpiops.comm.barrier()
log.debug('Finished Orbital error correction')
def test_orbital_correction_legacy_equality_orbfit_method_2(self):
remove_orbital_error(self.ifgs, self.params)
onlyfiles = [
f for f in os.listdir(SML_TEST_LEGACY_ORBITAL_DIR)
if os.path.isfile(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f))
and f.endswith('.csv') and f.__contains__('_method2_')
]
count = 0
for i, f in enumerate(onlyfiles):
legacy_phase_data = np.genfromtxt(os.path.join(
SML_TEST_LEGACY_ORBITAL_DIR, f),
delimiter=',')
for k, j in enumerate(self.ifgs):
if os.path.basename(j.data_path).split('_unw.')[0] == \
os.path.basename(f).split(
'_method2_')[1].split('.')[0]:
count += 1
# all numbers equal
np.testing.assert_array_almost_equal(legacy_phase_data,
j.phase_data,
decimal=3)
# number of nans must equal
self.assertEqual(np.sum(np.isnan(legacy_phase_data)),
np.sum(np.isnan(j.phase_data)))
# ensure that we have expected number of matches
self.assertEqual(count, len(self.ifgs))
def test_orbital_error_method2_dummy(self):
"""
does not test anything except that the method is working
"""
# change to orbital error correction method 2
self.params[cf.ORBITAL_FIT_METHOD] = NETWORK_METHOD
self.params[cf.ORBITAL_FIT_LOOKS_X] = 2
self.params[cf.ORBITAL_FIT_LOOKS_Y] = 2
correct._copy_mlooked(self.params)
correct._create_ifg_dict(self.params)
remove_orbital_error(self.new_data_paths, self.params)
onlyfiles = [f for f in os.listdir(SML_TEST_LEGACY_ORBITAL_DIR)
if os.path.isfile(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f))
and f.endswith('.csv') and f.__contains__('_method2_')]
count = 0
for i, f in enumerate(onlyfiles):
legacy_phase_data = np.genfromtxt(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f), delimiter=',')
for k, j in enumerate(self.new_data_paths):
if os.path.basename(j).split('_unw.')[0] == os.path.basename(f).split('_method2_')[1].split('.')[0]:
count += 1
ifg = Ifg(j)
ifg.open()
# number of nans must equal
assert np.sum(np.isnan(legacy_phase_data)) == np.sum(np.isnan(ifg.phase_data))
# ensure that we have expected number of matches
assert count == len(self.new_data_paths)
def test_orbital_correction_legacy_equality_orbfit_method_2(self):
correct._copy_mlooked(self.params)
correct._create_ifg_dict(self.params)
remove_orbital_error(self.new_data_paths, self.params)
onlyfiles = [f for f in os.listdir(SML_TEST_LEGACY_ORBITAL_DIR)
if os.path.isfile(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f))
and f.endswith('.csv') and f.__contains__('_method2_')]
count = 0
for i, f in enumerate(onlyfiles):
legacy_phase_data = np.genfromtxt(os.path.join(
SML_TEST_LEGACY_ORBITAL_DIR, f), delimiter=',')
for k, j in enumerate(self.new_data_paths):
if os.path.basename(j).split('_unw.')[0] == os.path.basename(f).split('_method2_')[1].split('.')[0]:
count += 1
ifg = Ifg(j)
ifg.open()
# all numbers equal
# Note this changed as the nodata mask in the gdal_python.gdal_average changed to nan from 0
# np.testing.assert_array_almost_equal(legacy_phase_data, ifg.phase_data, decimal=3)
# number of nans must equal
assert np.sum(np.isnan(legacy_phase_data)) == np.sum(np.isnan(ifg.phase_data))
# ensure that we have expected number of matches
assert count == len(self.new_data_paths)
def test_different_looks_raise(self):
# different x/y looks factors should be accepted
self.params[cf.ORBITAL_FIT_LOOKS_X] = 1
self.params[cf.ORBITAL_FIT_LOOKS_Y] = 5
try:
remove_orbital_error(self.ifgs, self.params)
except:
pytest.fail
def test_looks_as_int(self):
self.params[cf.ORBITAL_FIT_LOOKS_X] = 1.1
self.params[cf.ORBITAL_FIT_LOOKS_Y] = 5
with pytest.raises(OrbitalError):
remove_orbital_error(self.ifgs, self.params)
self.params[cf.ORBITAL_FIT_LOOKS_X] = 1
self.params[cf.ORBITAL_FIT_LOOKS_Y] = '5'
with pytest.raises(OrbitalError):
remove_orbital_error(self.ifgs, self.params)
def setup_class(cls):
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
conv2tif.main(params)
prepifg.main(params)
for p in params[cf.INTERFEROGRAM_FILES]: # hack
p.tmp_sampled_path = p.sampled_path
Path(p.sampled_path).chmod(
0o664
) # assign write permission as conv2tif output is readonly
params[cf.REF_EST_METHOD] = 2
params[cf.PARALLEL] = False
params[cf.ORBFIT_OFFSET] = True
dest_paths, headers = common.repair_params_for_correct_tests(
params[cf.OUT_DIR], params)
# start run_pyrate copy
ifgs = common.pre_prepare_ifgs(dest_paths, params)
mst_grid = common.mst_calculation(dest_paths, params)
# Estimate reference pixel location
refx, refy = ref_pixel_calc_wrapper(params)
# Estimate and remove orbit errors
remove_orbital_error(ifgs, params)
for i in ifgs:
i.close()
ifgs = common.pre_prepare_ifgs(dest_paths, params)
for i in ifgs:
i.close()
for p in params[cf.INTERFEROGRAM_FILES]:
p.tmp_sampled_path = p.sampled_path
params[cf.REFX], params[cf.REFY] = refx, refy
params['rows'], params['cols'] = 3, 2
correct._update_params_with_tiles(params)
cls.ref_phs, cls.ifgs = ref_phase_est_wrapper(params)
cls.params = params
def test_orb_error(self, orbfit_method, orbfit_degrees):
self.params[cf.ORBITAL_FIT_METHOD] = orbfit_method
self.params[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees
multi_paths = self.params[cf.INTERFEROGRAM_FILES]
self.ifg_paths = [p.tmp_sampled_path for p in multi_paths]
remove_orbital_error(self.ifg_paths, self.params)
# test_orb_errors_written
orb_error_files = [MultiplePaths.orb_error_path(i, self.params) for i in self.ifg_paths]
assert all(p.exists() for p in orb_error_files)
last_mod_times = np.array([os.stat(o).st_mtime for o in orb_error_files])
# run orbit removal again
remove_orbital_error(self.ifg_paths, self.params)
orb_error_files2 = [MultiplePaths.orb_error_path(i, self.params) for i in self.ifg_paths]
# if files are written again - times will change
last_mod_times_2 = np.array([os.stat(o).st_mtime for o in orb_error_files2])
# test_orb_error_reused_if_params_unchanged
assert all(a == b for a, b in zip(last_mod_times, last_mod_times_2))
# change one of the params
_degrees = set(cf.ORB_DEGREE_NAMES.keys())
_degrees.discard(orbfit_degrees)
# test_orb_errors_recalculated_if_params_change
self.params[cf.ORBITAL_FIT_DEGREE] = _degrees.pop()
remove_orbital_error(self.ifg_paths, self.params)
orb_error_files3 = [MultiplePaths.orb_error_path(i, self.params) for i in self.ifg_paths]
last_mod_times_3 = np.array([os.stat(o).st_mtime for o in orb_error_files3])
assert all(a != b for a, b in zip(last_mod_times, last_mod_times_3))
def test_orbital_error_method2_dummy(self):
"""
does not test anything except that the method is working
"""
# change to orbital error correction method 2
self.params[cf.ORBITAL_FIT_METHOD] = NETWORK_METHOD
self.params[cf.ORBITAL_FIT_LOOKS_X] = 2
self.params[cf.ORBITAL_FIT_LOOKS_Y] = 2
remove_orbital_error(self.ifgs, self.params)
onlyfiles = [
f for f in os.listdir(SML_TEST_LEGACY_ORBITAL_DIR)
if os.path.isfile(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f))
and f.endswith('.csv') and f.__contains__('_method2_')
]
count = 0
for i, f in enumerate(onlyfiles):
legacy_phase_data = np.genfromtxt(os.path.join(
SML_TEST_LEGACY_ORBITAL_DIR, f),
delimiter=',')
for k, j in enumerate(self.ifgs):
if os.path.basename(j.data_path).split('_unw.')[0] == \
os.path.basename(f).split(
'_method2_')[1].split('.')[0]:
count += 1
# # all numbers equal
# np.testing.assert_array_almost_equal(legacy_phase_data,
# j.phase_data, decimal=3)
#
# # number of nans must equal
# self.assertEqual(np.sum(np.isnan(legacy_phase_data)),
# np.sum(np.isnan(j.phase_data)))
# ensure that we have expected number of matches
self.assertEqual(count, len(self.ifgs))
def check_correction(self, degree, method, offset, decimal=2):
orig = array([c.phase_data.copy() for c in self.ifgs])
exp = [self.alt_orbital_correction(i, degree, offset) for i in self.ifgs]
params = dict()
params[cf.ORBITAL_FIT_METHOD] = method
params[cf.ORBITAL_FIT_DEGREE] = degree
params[cf.ORBFIT_OFFSET] = offset
params[cf.PARALLEL] = False
params[cf.NO_DATA_VALUE] = 0
params[cf.NAN_CONVERSION] = False
params[cf.OUT_DIR] = tempfile.mkdtemp()
params[cf.ORBITAL_FIT_LOOKS_X] = 1
params[cf.ORBITAL_FIT_LOOKS_Y] = 1
for i in self.ifgs:
i.mm_converted = True
remove_orbital_error(self.ifgs, params)
corrected = array([c.phase_data for c in self.ifgs])
assert ~(orig == corrected).all()
self.check_results(self.ifgs, orig) # test shape, data is non zero
# FIXME: is decimal=2 close enough?
for i, (e, a) in enumerate(zip(exp, corrected)):
assert_array_almost_equal(e, a, decimal=decimal)
def test_orb_error_multiple_run_does_not_change_phase_data(self, orbfit_method, orbfit_degrees):
self.params[cf.ORBITAL_FIT_METHOD] = orbfit_method
self.params[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees
remove_orbital_error(self.ifg_paths, self.params)
ifgs = [Ifg(i) for i in self.ifg_paths]
for i in ifgs:
i.open()
phase_prev = [i.phase_data for i in ifgs]
# orb correct once more
correct._copy_mlooked(self.params)
remove_orbital_error(self.ifg_paths, self.params)
# and again
correct._copy_mlooked(self.params)
remove_orbital_error(self.ifg_paths, self.params)
ifgs = [Ifg(i) for i in self.ifg_paths]
for i in ifgs:
i.open()
phase_now = [i.phase_data for i in ifgs]
np.testing.assert_array_equal(phase_now, phase_prev)
def test_invalid_method(self):
# test failure of a few different args for 'method'
for m in [None, 5, -1, -3, 45.8]:
self.params[cf.ORBITAL_FIT_METHOD] = m
with pytest.raises(OrbitalError):
remove_orbital_error(self.ifgs, self.params)
