def _orb_fit_calc(ifg_paths, params, preread_ifgs=None): """ MPI wrapper for orbital fit correction """ log.info('Calculating orbital correction') if not params[cf.ORBITAL_FIT]: log.info('Orbital correction not required') return if preread_ifgs: # don't check except for mpi tests # perform some general error/sanity checks log.debug('Checking Orbital error correction status') if mpiops.run_once(shared.check_correction_status, ifg_paths, ifc.PYRATE_ORBITAL_ERROR): log.debug('Finished Orbital error correction') return # return if True condition returned if params[cf.ORBITAL_FIT_METHOD] == 1: prcs_ifgs = mpiops.array_split(ifg_paths) orbital.remove_orbital_error(prcs_ifgs, params, preread_ifgs) else: # Here we do all the multilooking in one process, but in memory # can use multiple processes if we write data to disc during # remove_orbital_error step # A performance comparison should be made for saving multilooked # files on disc vs in memory single process multilooking if mpiops.rank == MASTER_PROCESS: orbital.remove_orbital_error(ifg_paths, params, preread_ifgs) mpiops.comm.barrier() log.debug('Finished Orbital error correction')
def test_orbital_correction_legacy_equality_orbfit_method_2(self): remove_orbital_error(self.ifgs, self.params) onlyfiles = [ f for f in os.listdir(SML_TEST_LEGACY_ORBITAL_DIR) if os.path.isfile(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f)) and f.endswith('.csv') and f.__contains__('_method2_') ] count = 0 for i, f in enumerate(onlyfiles): legacy_phase_data = np.genfromtxt(os.path.join( SML_TEST_LEGACY_ORBITAL_DIR, f), delimiter=',') for k, j in enumerate(self.ifgs): if os.path.basename(j.data_path).split('_unw.')[0] == \ os.path.basename(f).split( '_method2_')[1].split('.')[0]: count += 1 # all numbers equal np.testing.assert_array_almost_equal(legacy_phase_data, j.phase_data, decimal=3) # number of nans must equal self.assertEqual(np.sum(np.isnan(legacy_phase_data)), np.sum(np.isnan(j.phase_data))) # ensure that we have expected number of matches self.assertEqual(count, len(self.ifgs))
def test_orbital_error_method2_dummy(self): """ does not test anything except that the method is working """ # change to orbital error correction method 2 self.params[cf.ORBITAL_FIT_METHOD] = NETWORK_METHOD self.params[cf.ORBITAL_FIT_LOOKS_X] = 2 self.params[cf.ORBITAL_FIT_LOOKS_Y] = 2 correct._copy_mlooked(self.params) correct._create_ifg_dict(self.params) remove_orbital_error(self.new_data_paths, self.params) onlyfiles = [f for f in os.listdir(SML_TEST_LEGACY_ORBITAL_DIR) if os.path.isfile(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f)) and f.endswith('.csv') and f.__contains__('_method2_')] count = 0 for i, f in enumerate(onlyfiles): legacy_phase_data = np.genfromtxt(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f), delimiter=',') for k, j in enumerate(self.new_data_paths): if os.path.basename(j).split('_unw.')[0] == os.path.basename(f).split('_method2_')[1].split('.')[0]: count += 1 ifg = Ifg(j) ifg.open() # number of nans must equal assert np.sum(np.isnan(legacy_phase_data)) == np.sum(np.isnan(ifg.phase_data)) # ensure that we have expected number of matches assert count == len(self.new_data_paths)
def test_orbital_correction_legacy_equality_orbfit_method_2(self): correct._copy_mlooked(self.params) correct._create_ifg_dict(self.params) remove_orbital_error(self.new_data_paths, self.params) onlyfiles = [f for f in os.listdir(SML_TEST_LEGACY_ORBITAL_DIR) if os.path.isfile(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f)) and f.endswith('.csv') and f.__contains__('_method2_')] count = 0 for i, f in enumerate(onlyfiles): legacy_phase_data = np.genfromtxt(os.path.join( SML_TEST_LEGACY_ORBITAL_DIR, f), delimiter=',') for k, j in enumerate(self.new_data_paths): if os.path.basename(j).split('_unw.')[0] == os.path.basename(f).split('_method2_')[1].split('.')[0]: count += 1 ifg = Ifg(j) ifg.open() # all numbers equal # Note this changed as the nodata mask in the gdal_python.gdal_average changed to nan from 0 # np.testing.assert_array_almost_equal(legacy_phase_data, ifg.phase_data, decimal=3) # number of nans must equal assert np.sum(np.isnan(legacy_phase_data)) == np.sum(np.isnan(ifg.phase_data)) # ensure that we have expected number of matches assert count == len(self.new_data_paths)
def test_different_looks_raise(self): # different x/y looks factors should be accepted self.params[cf.ORBITAL_FIT_LOOKS_X] = 1 self.params[cf.ORBITAL_FIT_LOOKS_Y] = 5 try: remove_orbital_error(self.ifgs, self.params) except: pytest.fail
def test_looks_as_int(self): self.params[cf.ORBITAL_FIT_LOOKS_X] = 1.1 self.params[cf.ORBITAL_FIT_LOOKS_Y] = 5 with pytest.raises(OrbitalError): remove_orbital_error(self.ifgs, self.params) self.params[cf.ORBITAL_FIT_LOOKS_X] = 1 self.params[cf.ORBITAL_FIT_LOOKS_Y] = '5' with pytest.raises(OrbitalError): remove_orbital_error(self.ifgs, self.params)
def setup_class(cls): params = Configuration(common.TEST_CONF_ROIPAC).__dict__ conv2tif.main(params) prepifg.main(params) for p in params[cf.INTERFEROGRAM_FILES]: # hack p.tmp_sampled_path = p.sampled_path Path(p.sampled_path).chmod( 0o664 ) # assign write permission as conv2tif output is readonly params[cf.REF_EST_METHOD] = 2 params[cf.PARALLEL] = False params[cf.ORBFIT_OFFSET] = True dest_paths, headers = common.repair_params_for_correct_tests( params[cf.OUT_DIR], params) # start run_pyrate copy ifgs = common.pre_prepare_ifgs(dest_paths, params) mst_grid = common.mst_calculation(dest_paths, params) # Estimate reference pixel location refx, refy = ref_pixel_calc_wrapper(params) # Estimate and remove orbit errors remove_orbital_error(ifgs, params) for i in ifgs: i.close() ifgs = common.pre_prepare_ifgs(dest_paths, params) for i in ifgs: i.close() for p in params[cf.INTERFEROGRAM_FILES]: p.tmp_sampled_path = p.sampled_path params[cf.REFX], params[cf.REFY] = refx, refy params['rows'], params['cols'] = 3, 2 correct._update_params_with_tiles(params) cls.ref_phs, cls.ifgs = ref_phase_est_wrapper(params) cls.params = params
def test_orb_error(self, orbfit_method, orbfit_degrees): self.params[cf.ORBITAL_FIT_METHOD] = orbfit_method self.params[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees multi_paths = self.params[cf.INTERFEROGRAM_FILES] self.ifg_paths = [p.tmp_sampled_path for p in multi_paths] remove_orbital_error(self.ifg_paths, self.params) # test_orb_errors_written orb_error_files = [MultiplePaths.orb_error_path(i, self.params) for i in self.ifg_paths] assert all(p.exists() for p in orb_error_files) last_mod_times = np.array([os.stat(o).st_mtime for o in orb_error_files]) # run orbit removal again remove_orbital_error(self.ifg_paths, self.params) orb_error_files2 = [MultiplePaths.orb_error_path(i, self.params) for i in self.ifg_paths] # if files are written again - times will change last_mod_times_2 = np.array([os.stat(o).st_mtime for o in orb_error_files2]) # test_orb_error_reused_if_params_unchanged assert all(a == b for a, b in zip(last_mod_times, last_mod_times_2)) # change one of the params _degrees = set(cf.ORB_DEGREE_NAMES.keys()) _degrees.discard(orbfit_degrees) # test_orb_errors_recalculated_if_params_change self.params[cf.ORBITAL_FIT_DEGREE] = _degrees.pop() remove_orbital_error(self.ifg_paths, self.params) orb_error_files3 = [MultiplePaths.orb_error_path(i, self.params) for i in self.ifg_paths] last_mod_times_3 = np.array([os.stat(o).st_mtime for o in orb_error_files3]) assert all(a != b for a, b in zip(last_mod_times, last_mod_times_3))
def test_orbital_error_method2_dummy(self): """ does not test anything except that the method is working """ # change to orbital error correction method 2 self.params[cf.ORBITAL_FIT_METHOD] = NETWORK_METHOD self.params[cf.ORBITAL_FIT_LOOKS_X] = 2 self.params[cf.ORBITAL_FIT_LOOKS_Y] = 2 remove_orbital_error(self.ifgs, self.params) onlyfiles = [ f for f in os.listdir(SML_TEST_LEGACY_ORBITAL_DIR) if os.path.isfile(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f)) and f.endswith('.csv') and f.__contains__('_method2_') ] count = 0 for i, f in enumerate(onlyfiles): legacy_phase_data = np.genfromtxt(os.path.join( SML_TEST_LEGACY_ORBITAL_DIR, f), delimiter=',') for k, j in enumerate(self.ifgs): if os.path.basename(j.data_path).split('_unw.')[0] == \ os.path.basename(f).split( '_method2_')[1].split('.')[0]: count += 1 # # all numbers equal # np.testing.assert_array_almost_equal(legacy_phase_data, # j.phase_data, decimal=3) # # # number of nans must equal # self.assertEqual(np.sum(np.isnan(legacy_phase_data)), # np.sum(np.isnan(j.phase_data))) # ensure that we have expected number of matches self.assertEqual(count, len(self.ifgs))
def check_correction(self, degree, method, offset, decimal=2): orig = array([c.phase_data.copy() for c in self.ifgs]) exp = [self.alt_orbital_correction(i, degree, offset) for i in self.ifgs] params = dict() params[cf.ORBITAL_FIT_METHOD] = method params[cf.ORBITAL_FIT_DEGREE] = degree params[cf.ORBFIT_OFFSET] = offset params[cf.PARALLEL] = False params[cf.NO_DATA_VALUE] = 0 params[cf.NAN_CONVERSION] = False params[cf.OUT_DIR] = tempfile.mkdtemp() params[cf.ORBITAL_FIT_LOOKS_X] = 1 params[cf.ORBITAL_FIT_LOOKS_Y] = 1 for i in self.ifgs: i.mm_converted = True remove_orbital_error(self.ifgs, params) corrected = array([c.phase_data for c in self.ifgs]) assert ~(orig == corrected).all() self.check_results(self.ifgs, orig) # test shape, data is non zero # FIXME: is decimal=2 close enough? for i, (e, a) in enumerate(zip(exp, corrected)): assert_array_almost_equal(e, a, decimal=decimal)
def test_orb_error_multiple_run_does_not_change_phase_data(self, orbfit_method, orbfit_degrees): self.params[cf.ORBITAL_FIT_METHOD] = orbfit_method self.params[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees remove_orbital_error(self.ifg_paths, self.params) ifgs = [Ifg(i) for i in self.ifg_paths] for i in ifgs: i.open() phase_prev = [i.phase_data for i in ifgs] # orb correct once more correct._copy_mlooked(self.params) remove_orbital_error(self.ifg_paths, self.params) # and again correct._copy_mlooked(self.params) remove_orbital_error(self.ifg_paths, self.params) ifgs = [Ifg(i) for i in self.ifg_paths] for i in ifgs: i.open() phase_now = [i.phase_data for i in ifgs] np.testing.assert_array_equal(phase_now, phase_prev)
def test_invalid_method(self): # test failure of a few different args for 'method' for m in [None, 5, -1, -3, 45.8]: self.params[cf.ORBITAL_FIT_METHOD] = m with pytest.raises(OrbitalError): remove_orbital_error(self.ifgs, self.params)