def test_mixed_metadata_raises(self): # change config to 5 ifgs self.params[cf.INTERFEROGRAM_FILES] = [ MultiplePaths(p, self.params) for p in self.small_tifs[:5] ] for p in self.params[cf.INTERFEROGRAM_FILES]: p.sampled_path = p.converted_path p.tmp_sampled_path = p.sampled_path # correct reference phase for some of the ifgs ref_phase_est_wrapper(self.params) for ifg in self.ifgs: ifg.open() # change config to all ifgs self.params[cf.INTERFEROGRAM_FILES] = [ MultiplePaths(p, self.params) for p in self.small_tifs ] for p in self.params[cf.INTERFEROGRAM_FILES]: p.sampled_path = p.converted_path p.tmp_sampled_path = p.sampled_path # now it should raise exception if we want to correct refernece phase again on all of them with pytest.raises(CorrectionStatusError): ref_phase_est_wrapper(self.params)
def test_orb_error(self, orbfit_method, orbfit_degrees): self.params[cf.ORBITAL_FIT_METHOD] = orbfit_method self.params[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees multi_paths = self.params[cf.INTERFEROGRAM_FILES] self.ifg_paths = [p.tmp_sampled_path for p in multi_paths] remove_orbital_error(self.ifg_paths, self.params) # test_orb_errors_written orb_error_files = [MultiplePaths.orb_error_path(i, self.params) for i in self.ifg_paths] assert all(p.exists() for p in orb_error_files) last_mod_times = np.array([os.stat(o).st_mtime for o in orb_error_files]) # run orbit removal again remove_orbital_error(self.ifg_paths, self.params) orb_error_files2 = [MultiplePaths.orb_error_path(i, self.params) for i in self.ifg_paths] # if files are written again - times will change last_mod_times_2 = np.array([os.stat(o).st_mtime for o in orb_error_files2]) # test_orb_error_reused_if_params_unchanged assert all(a == b for a, b in zip(last_mod_times, last_mod_times_2)) # change one of the params _degrees = set(cf.ORB_DEGREE_NAMES.keys()) _degrees.discard(orbfit_degrees) # test_orb_errors_recalculated_if_params_change self.params[cf.ORBITAL_FIT_DEGREE] = _degrees.pop() remove_orbital_error(self.ifg_paths, self.params) orb_error_files3 = [MultiplePaths.orb_error_path(i, self.params) for i in self.ifg_paths] last_mod_times_3 = np.array([os.stat(o).st_mtime for o in orb_error_files3]) assert all(a != b for a, b in zip(last_mod_times, last_mod_times_3))
def test_multilooked_tiffs_converted_to_unw_are_same(self): # Get multilooked geotiffs geotiffs = list(set(self.dest_paths)) geotiffs = [g for g in geotiffs if 'dem' not in g] # Convert back to .unw dest_unws = [] for g in set(geotiffs): dest_unw = os.path.join(self.params[cf.OUT_DIR], Path(g).stem + '.unw') shared.write_unw_from_data_or_geotiff(geotif_or_data=g, dest_unw=dest_unw, ifg_proc=1) dest_unws.append(dest_unw) dest_unws_ = [] for d in dest_unws: dest_unws_.append(MultiplePaths(d, self.params)) # Convert back to tiff new_geotiffs_ = conv2tif.do_geotiff(dest_unws_, self.params) new_geotiffs = [gt for gt, b in new_geotiffs_] # Ensure original multilooked geotiffs and # unw back to geotiff are the same geotiffs.sort() new_geotiffs.sort() for g, u in zip(geotiffs, new_geotiffs): g_ds = gdal.Open(g) u_gs = gdal.Open(u) np.testing.assert_array_almost_equal(u_gs.ReadAsArray(), g_ds.ReadAsArray()) u_gs = None g_ds = None
def _ts_to_ifgs(tsincr, preread_ifgs, params): """ Function that converts an incremental displacement time series into interferometric phase observations. Used to re-construct an interferogram network from a time series. :param ndarray tsincr: incremental time series array of size (ifg.shape, nepochs-1) :param dict preread_ifgs: Dictionary of shared.PrereadIfg class instances :return: None, interferograms are saved to disk """ log.debug('Reconstructing interferometric observations from time series') ifgs = list(OrderedDict(sorted(preread_ifgs.items())).values()) _, n = mpiops.run_once(get_epochs, ifgs) index_first, index_second = n[:len(ifgs)], n[len(ifgs):] num_ifgs_tuples = mpiops.array_split(list(enumerate(ifgs))) num_ifgs_tuples = [(int(num), ifg) for num, ifg in num_ifgs_tuples] for i, ifg in num_ifgs_tuples: aps_correction_on_disc = MultiplePaths.aps_error_path( ifg.tmp_path, params) phase = np.sum(tsincr[:, :, index_first[i]:index_second[i]], axis=2) np.save(file=aps_correction_on_disc, arr=phase) _save_aps_corrected_phase(ifg.tmp_path, phase)
def test_multilooked_tiffs_converted_to_unw_are_same(self): # Get multilooked geotiffs geotiffs = self.dest_paths # Convert back to .unw dest_unws = [] for g in geotiffs: dest_unw = os.path.join(self.params[cf.OUT_DIR], os.path.splitext(g)[0] + '.unw') shared.write_unw_from_data_or_geotiff(geotif_or_data=g, dest_unw=dest_unw, ifg_proc=1) dest_unws.append(dest_unw) dest_unws_ = [ MultiplePaths(self.params[cf.OUT_DIR], b, self.params[cf.IFG_LKSX], self.params[cf.IFG_CROP_OPT]) for b in dest_unws ] # Convert back to tiff new_geotiffs_ = conv2tif.do_geotiff(dest_unws_, self.params) new_geotiffs = [gt for gt, b in new_geotiffs_] # Ensure original multilooked geotiffs and # unw back to geotiff are the same for g, u in zip(geotiffs, new_geotiffs): g_ds = gdal.Open(g) u_gs = gdal.Open(u) np.testing.assert_array_almost_equal(u_gs.ReadAsArray(), g_ds.ReadAsArray()) u_gs = None g_ds = None
def setup_method(self): self.params = Configuration(common.TEST_CONF_GAMMA).__dict__ self.tmp_dir = tempfile.mkdtemp() self.params[cf.OUT_DIR] = self.tmp_dir self.params[cf.REF_EST_METHOD] = 1 self.params[cf.PARALLEL] = False self.params[cf.TMPDIR] = self.tmp_dir common.copytree(common.SML_TEST_TIF, self.tmp_dir) self.small_tifs = glob.glob(os.path.join(self.tmp_dir, "*.tif")) for s in self.small_tifs: os.chmod(s, 0o644) self.ifgs = common.small_data_setup(self.tmp_dir, is_dir=True) self.params[cf.INTERFEROGRAM_FILES] = [ MultiplePaths(p, self.params) for p in self.small_tifs ] for p in self.params[cf.INTERFEROGRAM_FILES]: p.sampled_path = p.converted_path p.tmp_sampled_path = p.sampled_path for ifg in self.ifgs: ifg.close() self.params[cf.REFX], self.params[cf.REFY] = -1, -1 self.params[cf.REFNX], self.params[cf.REFNY] = 10, 10 self.params[cf.REF_CHIP_SIZE], self.params[cf.REF_MIN_FRAC] = 21, 0.5 self.params['rows'], self.params['cols'] = 3, 2 self.params[cf.REF_PIXEL_FILE] = Configuration.ref_pixel_path( self.params) correct._update_params_with_tiles(self.params) correct.ref_pixel_calc_wrapper(self.params)
def __check_and_apply_demerrors_found_on_disc(ifg_paths: list, params: dict) -> bool: """ Convenience function to check if DEM error correction files have already been produced in a previous run :param ifg_paths: List of interferogram class objects. :param params: Dictionary of PyRate configuration parameters. :return: bool value: True if dem error files found on disc, otherwise False. """ saved_dem_err_paths = [ MultiplePaths.dem_error_path(ifg_path, params) for ifg_path in ifg_paths ] for d, i in zip(saved_dem_err_paths, ifg_paths): if d.exists(): dem_corr = np.load(d) if isinstance(i, str): # are paths ifg = Ifg(i) ifg.open() else: ifg = i ifg.phase_data -= dem_corr # set geotiff meta tag and save phase to file # TODO: calculate avg bperp and add to metadata even for reused DEM error correction _save_dem_error_corrected_phase(ifg) return all(d.exists() for d in saved_dem_err_paths)
def spatio_temporal_filter(params: dict) -> None: """ Applies a spatio-temporal filter to remove the atmospheric phase screen (APS) and saves the corrected interferograms. Firstly the incremental time series is computed using the SVD method, before a cascade of temporal then spatial Gaussian filters is applied. The resulting APS corrections are saved to disc before being subtracted from each interferogram. :param params: Dictionary of PyRate configuration parameters. """ if params[C.APSEST]: log.info('Doing APS spatio-temporal filtering') else: log.info('APS spatio-temporal filtering not required') return tiles = params[C.TILES] preread_ifgs = params[C.PREREAD_IFGS] ifg_paths = [ ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES] ] # perform some checks on existing ifgs log.debug('Checking APS correction status') if mpiops.run_once(shared.check_correction_status, ifg_paths, ifc.PYRATE_APS_ERROR): log.debug('Finished APS correction') return # return if True condition returned aps_paths = [MultiplePaths.aps_error_path(i, params) for i in ifg_paths] if all(a.exists() for a in aps_paths): log.warning('Reusing APS errors from previous run') _apply_aps_correction(ifg_paths, aps_paths, params) return # obtain the incremental time series using SVD tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles) mpiops.comm.barrier() # get lists of epochs and ifgs ifgs = list(OrderedDict(sorted(preread_ifgs.items())).values()) epochlist = mpiops.run_once(get_epochs, ifgs)[0] # first perform temporal high pass filter ts_hp = temporal_high_pass_filter(tsincr, epochlist, params) # second perform spatial low pass filter to obtain APS correction in ts domain ifg = Ifg(ifg_paths[0]) # just grab any for parameters in slpfilter ifg.open() ts_aps = spatial_low_pass_filter(ts_hp, ifg, params) ifg.close() # construct APS corrections for each ifg _make_aps_corrections(ts_aps, ifgs, params) # apply correction to ifgs and save ifgs to disc. _apply_aps_correction(ifg_paths, aps_paths, params) # update/save the phase_data in the tiled numpy files shared.save_numpy_phase(ifg_paths, params)
def _write_dem_errors(ifg_paths: list, params: dict, preread_ifgs: dict) -> None: """ Convenience function for writing DEM error (one file) and DEM error correction for each IFG to disc :param ifg_paths: List of interferogram class objects. :param params: Dictionary of PyRate configuration parameters. :param preread_ifgs: Dictionary of interferogram metadata. """ tiles = params[C.TILES] # re-assemble tiles and save into dem_error dir shape = preread_ifgs[ifg_paths[0]].shape # save dem error as geotiff file in out directory gt, md, wkt = shared.get_geotiff_header_info(ifg_paths[0]) md[ifc.DATA_TYPE] = ifc.DEM_ERROR dem_error = assemble_tiles(shape, params[C.TMPDIR], tiles, out_type='dem_error') dem_error_file = os.path.join(params[C.DEM_ERROR_DIR], 'dem_error.tif') shared.remove_file_if_exists(dem_error_file) shared.write_output_geotiff(md, gt, wkt, dem_error, dem_error_file, np.nan) # read the average bperp vals for each ifg and each tile bperp = np.empty(shape=(len(tiles), len(ifg_paths)), dtype=np.float64) for t in tiles: bperp_file = Path( join(params[C.TMPDIR], 'bperp_avg_' + str(t.index) + '.npy')) arr = np.load(file=bperp_file) bperp[t.index, :] = arr # loop over all ifgs idx = 0 for ifg_path in ifg_paths: ifg = Ifg(ifg_path) ifg.open() shared.nan_and_mm_convert(ifg, params) # ensure we have phase data in mm # read dem error correction file from tmpdir dem_error_correction_ifg = assemble_tiles( shape, params[C.TMPDIR], tiles, out_type='dem_error_correction', index=idx) # calculate average bperp value across all tiles for the ifg bperp_val = np.nanmean(bperp[:, idx]) dem_error_correction_on_disc = MultiplePaths.dem_error_path( ifg.data_path, params) np.save(file=dem_error_correction_on_disc, arr=dem_error_correction_ifg) idx += 1 # subtract DEM error from the ifg ifg.phase_data -= dem_error_correction_ifg _save_dem_error_corrected_phase(ifg, bperp_val)
def independent_orbital_correction(ifg, params): """ Calculates and removes an orbital error surface from a single independent interferogram. Warning: This will write orbital error corrected phase_data to the ifg. :param Ifg class instance ifg: the interferogram to be corrected :param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC) :param bool offset: True to calculate the model using an offset :param dict params: dictionary of configuration parameters :return: None - interferogram phase data is updated and saved to disk """ degree = params[cf.ORBITAL_FIT_DEGREE] offset = params[cf.ORBFIT_OFFSET] orbfit_correction_on_disc = MultiplePaths.orb_error_path( ifg.data_path, params) if not ifg.is_open: ifg.open() shared.nan_and_mm_convert(ifg, params) if orbfit_correction_on_disc.exists(): log.info( f'Reusing already computed orbital fit correction for {ifg.data_path}' ) orbital_correction = np.load(file=orbfit_correction_on_disc) else: # vectorise, keeping NODATA vphase = reshape(ifg.phase_data, ifg.num_cells) dm = get_design_matrix(ifg, degree, offset) # filter NaNs out before getting model clean_dm = dm[~isnan(vphase)] data = vphase[~isnan(vphase)] model = lstsq(clean_dm, data)[0] # first arg is the model params # calculate forward model & morph back to 2D if offset: fullorb = np.reshape(np.dot(dm[:, :-1], model[:-1]), ifg.phase_data.shape) else: fullorb = np.reshape(np.dot(dm, model), ifg.phase_data.shape) if not orbfit_correction_on_disc.parent.exists(): shared.mkdir_p(orbfit_correction_on_disc.parent) offset_removal = nanmedian(np.ravel(ifg.phase_data - fullorb)) orbital_correction = fullorb - offset_removal # dump to disc np.save(file=orbfit_correction_on_disc, arr=orbital_correction) # subtract orbital error from the ifg ifg.phase_data -= orbital_correction # set orbfit meta tag and save phase to file _save_orbital_error_corrected_phase(ifg) ifg.close()
def setup_class(cls): # testing constants2 cls.BASE_DIR = tempfile.mkdtemp() cls.BASE_OUT_DIR = join(cls.BASE_DIR, 'out') cls.BASE_DEM_DIR = join(cls.BASE_DIR, 'dem') cls.BASE_DEM_FILE = join(cls.BASE_DEM_DIR, 'roipac_test_trimmed.tif') try: # copy source data (treat as prepifg already run) os.makedirs(cls.BASE_OUT_DIR) for path in glob.glob(join(common.SML_TEST_TIF, '*')): dest = join(cls.BASE_OUT_DIR, os.path.basename(path)) shutil.copy(path, dest) os.chmod(dest, 0o660) os.makedirs(cls.BASE_DEM_DIR) orig_dem = common.SML_TEST_DEM_TIF os.symlink(orig_dem, cls.BASE_DEM_FILE) os.chdir(cls.BASE_DIR) # Turn off validation because we're in a different working dir # and relative paths in config won't be work. params = config.get_config_params(common.TEST_CONF_ROIPAC) params['correct'] = [ 'orbfit', 'refphase', 'mst', 'apscorrect', 'maxvar' ] params[cf.OUT_DIR] = cls.BASE_OUT_DIR params[cf.PROCESSOR] = 0 # roipac params[cf.APS_CORRECTION] = 0 paths = glob.glob(join(cls.BASE_OUT_DIR, 'geo_*-*.tif')) paths = sorted(paths) params[cf.PARALLEL] = False params[cf.ORBFIT_OFFSET] = True params[cf.TEMP_MLOOKED_DIR] = cls.BASE_OUT_DIR.join( cf.TEMP_MLOOKED_DIR) params[cf.INTERFEROGRAM_FILES] = [ MultiplePaths(p, params) for p in paths ] for p in params[cf.INTERFEROGRAM_FILES]: # cheat p.sampled_path = p.converted_path p.tmp_sampled_path = p.converted_path params["rows"], params["cols"] = 2, 2 params[cf.REF_PIXEL_FILE] = Configuration.ref_pixel_path(params) Path(params[cf.OUT_DIR]).joinpath(cf.APS_ERROR_DIR).mkdir( exist_ok=True, parents=True) Path(params[cf.OUT_DIR]).joinpath(cf.MST_DIR).mkdir(exist_ok=True, parents=True) correct.correct_ifgs(params) if not hasattr(cls, 'ifgs'): cls.ifgs = get_ifgs(out_dir=cls.BASE_OUT_DIR) except: # revert working dir & avoid paths busting other tests os.chdir(CURRENT_DIR) raise
def test_need_at_least_two_ifgs(self): self.params[cf.INTERFEROGRAM_FILES] = [ MultiplePaths(p, self.params) for p in self.small_tifs[:1] ] for p in self.params[cf.INTERFEROGRAM_FILES]: p.sampled_path = p.converted_path p.tmp_sampled_path = p.sampled_path with pytest.raises(ReferencePhaseError): ref_phase_est_wrapper(self.params)
def setup_class(cls): roipac_params = Configuration(TEST_CONF_ROIPAC).__dict__ from copy import deepcopy params = deepcopy(roipac_params) shared.mkdir_p(params[C.TMPDIR]) params[C.REF_EST_METHOD] = 2 conv2tif.main(params) params = deepcopy(roipac_params) prepifg.main(params) params = deepcopy(roipac_params) base_ifg_paths = [ c.unwrapped_path for c in params[C.INTERFEROGRAM_FILES] ] dest_paths = [c.converted_path for c in params[C.INTERFEROGRAM_FILES]] params[C.INTERFEROGRAM_FILES] = [ MultiplePaths(d, params) for d in dest_paths ] for p in params[C.INTERFEROGRAM_FILES]: # hack p.sampled_path = p.converted_path for i in dest_paths: Path(i).chmod( 0o664 ) # assign write permission as conv2tif output is readonly ifgs = common.pre_prepare_ifgs(dest_paths, params) correct._copy_mlooked(params) correct._update_params_with_tiles(params) correct._create_ifg_dict(params) pyrate.core.refpixel.ref_pixel_calc_wrapper(params) params[C.ORBFIT_OFFSET] = True pyrate.core.orbital.remove_orbital_error(ifgs, params) ifgs = prepare_ifgs_without_phase(dest_paths, params) for ifg in ifgs: ifg.close() for p in params[C.INTERFEROGRAM_FILES]: # hack p.tmp_sampled_path = p.sampled_path _, cls.ifgs = pyrate.core.ref_phs_est.ref_phase_est_wrapper(params) ifgs[0].open() r_dist = RDist(ifgs[0])() ifgs[0].close() # Calculate interferogram noise cls.maxvar = [ cvd(i, params, r_dist, calc_alpha=True, save_acg=True, write_vals=True)[0] for i in dest_paths ] cls.vcmt = get_vcmt(ifgs, cls.maxvar) for ifg in ifgs: ifg.close() cls.params = params
def __run_once(self): dem_files = [ MultiplePaths.dem_error_path(i, self.params) for i in self.ifg_paths ] correct._copy_mlooked(self.params) correct._update_params_with_tiles(self.params) correct._create_ifg_dict(self.params) save_numpy_phase(self.ifg_paths, self.params) dem_error_calc_wrapper(self.params) assert all(m.exists() for m in dem_files) return [os.stat(o).st_mtime for o in dem_files]
def test_aps_error_files_on_disc(self, slpfmethod, slpfcutoff, slpforder): self.params[cf.SLPF_METHOD] = slpfmethod self.params[cf.SLPF_CUTOFF] = slpfcutoff self.params[cf.SLPF_ORDER] = slpforder wrap_spatio_temporal_filter(self.params) # test_orb_errors_written aps_error_files = [ MultiplePaths.aps_error_path(i, self.params) for i in self.ifg_paths ] assert all(p.exists() for p in aps_error_files) last_mod_times = [os.stat(o).st_mtime for o in aps_error_files] phase_prev = [i.phase_data for i in self.ifgs] # run aps error removal again wrap_spatio_temporal_filter(self.params) aps_error_files2 = [ MultiplePaths.aps_error_path(i, self.params) for i in self.ifg_paths ] # if files are written again - times will change last_mod_times_2 = [os.stat(o).st_mtime for o in aps_error_files2] # test_aps_error_reused_if_params_unchanged assert all(a == b for a, b in zip(last_mod_times, last_mod_times_2)) phase_now = [i.phase_data for i in self.ifgs] # run aps error correction once mroe wrap_spatio_temporal_filter(self.params) aps_error_files3 = [ MultiplePaths.aps_error_path(i, self.params) for i in self.ifg_paths ] last_mod_times_3 = [os.stat(o).st_mtime for o in aps_error_files3] assert all(a == b for a, b in zip(last_mod_times, last_mod_times_3)) phase_again = [i.phase_data for i in self.ifgs] np.testing.assert_array_equal(phase_prev, phase_now) np.testing.assert_array_equal(phase_prev, phase_again)
def prepare_ifgs(raster_data_paths, crop_opt, xlooks, ylooks, headers, params, thresh=0.5, user_exts=None, write_to_disc=True): """ Wrapper function to prepare a sequence of interferogram files for PyRate analysis. See prepifg.prepare_ifg() for full description of inputs and returns. Note: function need refining for crop options :param list raster_data_paths: List of interferogram file paths :param int crop_opt: Crop option :param int xlooks: Number of multi-looks in x; 5 is 5 times smaller, 1 is no change :param int ylooks: Number of multi-looks in y :param float thresh: see thresh in prepare_ifgs() :param tuple user_exts: Tuple of user defined georeferenced extents for new file: (xfirst, yfirst, xlast, ylast)cropping coordinates :param bool write_to_disc: Write new data to disk :return: resampled_data: output cropped and resampled image :rtype: ndarray :return: out_ds: destination gdal dataset object :rtype: List[gdal.Dataset] """ if xlooks != ylooks: log.warning('X and Y multi-look factors are not equal') # use metadata check to check whether it's a dem or ifg rasters = [dem_or_ifg(r) for r in raster_data_paths] exts = get_analysis_extent(crop_opt, rasters, xlooks, ylooks, user_exts) out_paths = [] for r, t in zip(raster_data_paths, rasters): if isinstance(t, DEM): input_type = InputTypes.DEM else: input_type = InputTypes.IFG out_path = MultiplePaths(r, params, input_type).sampled_path out_paths.append(out_path) return [ prepare_ifg(d, xlooks, ylooks, exts, thresh, crop_opt, h, write_to_disc, p) for d, h, p in zip(raster_data_paths, headers, out_paths) ]
def _remove_network_orb_error(coefs, dm, ifg, ids, offset, params): """ remove network orbital error from input interferograms """ saved_orb_err_path = MultiplePaths.orb_error_path(ifg.data_path, params) orb = dm.dot(coefs[ids[ifg.second]] - coefs[ids[ifg.first]]) orb = orb.reshape(ifg.shape) # offset estimation if offset: # bring all ifgs to same base level orb -= nanmedian(np.ravel(ifg.phase_data - orb)) # subtract orbital error from the ifg ifg.phase_data -= orb # save orb error on disc np.save(file=saved_orb_err_path, arr=orb) # set orbfit meta tag and save phase to file _save_orbital_error_corrected_phase(ifg)
def test_orbital_correction_legacy_equality(self): from pyrate import correct from pyrate.configuration import MultiplePaths multi_paths = [MultiplePaths(p, params=self.params) for p in self.ifg_paths] for m in multi_paths: # cheat m.sampled_path = m.converted_path self.params[cf.INTERFEROGRAM_FILES] = multi_paths self.params['rows'], self.params['cols'] = 2, 3 Path(self.BASE_DIR).joinpath('tmpdir').mkdir(exist_ok=True, parents=True) correct._copy_mlooked(self.params) correct._update_params_with_tiles(self.params) correct._create_ifg_dict(self.params) correct._copy_mlooked(self.params) pyrate.core.orbital.orb_fit_calc_wrapper(self.params) onlyfiles = [f for f in os.listdir(SML_TEST_LEGACY_ORBITAL_DIR) if os.path.isfile(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f)) and f.endswith('.csv') and f.__contains__('_method1_')] count = 0 for i, f in enumerate(onlyfiles): ifg_data = np.genfromtxt(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f), delimiter=',') for k, j in enumerate([m.tmp_sampled_path for m in multi_paths]): ifg = Ifg(j) ifg.open() print(f) print(j) if os.path.basename(j).split('_ifg.')[0] == os.path.basename(f).split( '_orb_planar_1lks_method1_geo_')[1].split('.')[0]: count += 1 # all numbers equal np.testing.assert_array_almost_equal(ifg_data, ifg.phase_data, decimal=2) # means must also be equal assert np.nanmean(ifg_data) == pytest.approx(np.nanmean(ifg.phase_data), abs=1e-2) # number of nans must equal assert np.sum(np.isnan(ifg_data)) == np.sum(np.isnan(ifg.phase_data)) ifg.close() # ensure that we have expected number of matches assert count == len(self.ifg_paths)
def __check_and_apply_orberrors_found_on_disc(ifg_paths, params): saved_orb_err_paths = [ MultiplePaths.orb_error_path(ifg_path, params) for ifg_path in ifg_paths ] for p, i in zip(saved_orb_err_paths, ifg_paths): if p.exists(): orb = np.load(p) if isinstance(i, str): # are paths ifg = Ifg(i) ifg.open(readonly=False) shared.nan_and_mm_convert(ifg, params) else: ifg = i ifg.phase_data -= orb # set orbfit meta tag and save phase to file _save_orbital_error_corrected_phase(ifg) return all(p.exists() for p in saved_orb_err_paths)
def _remove_network_orb_error(coefs, dm, ifg, ids, offset, params): """ remove network orbital error from input interferograms """ saved_orb_err_path = MultiplePaths.orb_error_path(ifg.data_path, params) orb = dm.dot(coefs[ids[ifg.second]] - coefs[ids[ifg.first]]) orb = orb.reshape(ifg.shape) # Estimate the offset of the interferogram as the median of ifg minus model # Only needed if reference phase correction has already been applied? if offset: # brings all ifgs to same reference level orb -= nanmedian(np.ravel(ifg.phase_data - orb)) # subtract orbital error from the ifg ifg.phase_data -= orb # save orb error on disc np.save(file=saved_orb_err_path, arr=orb) # set orbfit meta tag and save phase to file _save_orbital_error_corrected_phase(ifg, params)
def test_orbital_correction_legacy_equality(self): from pyrate import process from pyrate.configuration import MultiplePaths multi_paths = [MultiplePaths(self.BASE_DIR, p) for p in self.ifg_paths] for m in multi_paths: # cheat m.sampled_path = m.converted_path process._orb_fit_calc(multi_paths, self.params) onlyfiles = [ f for f in os.listdir(SML_TEST_LEGACY_ORBITAL_DIR) if os.path.isfile(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f)) and f.endswith('.csv') and f.__contains__('_method1_') ] count = 0 for i, f in enumerate(onlyfiles): ifg_data = np.genfromtxt(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f), delimiter=',') for k, j in enumerate(self.ifg_paths): ifg = Ifg(j) ifg.open() if os.path.basename(j).split('_unw.')[0] == os.path.basename( f).split('_orb_planar_1lks_method1_')[1].split('.')[0]: count += 1 # all numbers equal np.testing.assert_array_almost_equal(ifg_data, ifg.phase_data, decimal=2) # means must also be equal self.assertAlmostEqual(np.nanmean(ifg_data), np.nanmean(ifg.phase_data), places=2) # number of nans must equal self.assertEqual(np.sum(np.isnan(ifg_data)), np.sum(np.isnan(ifg.phase_data))) ifg.close() # ensure that we have expected number of matches self.assertEqual(count, len(self.ifg_paths))
def setUpClass(cls): # testing constants2 cls.BASE_DIR = tempfile.mkdtemp() cls.BASE_OUT_DIR = join(cls.BASE_DIR, 'out') cls.BASE_DEM_DIR = join(cls.BASE_DIR, 'dem') cls.BASE_DEM_FILE = join(cls.BASE_DEM_DIR, 'roipac_test_trimmed.tif') try: # copy source data (treat as prepifg already run) os.makedirs(cls.BASE_OUT_DIR) for path in glob.glob(join(common.SML_TEST_TIF, '*')): dest = join(cls.BASE_OUT_DIR, os.path.basename(path)) shutil.copy(path, dest) os.chmod(dest, 0o660) os.makedirs(cls.BASE_DEM_DIR) orig_dem = common.SML_TEST_DEM_TIF os.symlink(orig_dem, cls.BASE_DEM_FILE) os.chdir(cls.BASE_DIR) # Turn off validation because we're in a different working dir # and relative paths in config won't be work. params = config.get_config_params(common.TEST_CONF_ROIPAC) params[cf.OUT_DIR] = cls.BASE_OUT_DIR params[cf.PROCESSOR] = 0 # roipac params[cf.APS_CORRECTION] = 0 paths = glob.glob(join(cls.BASE_OUT_DIR, 'geo_*-*.tif')) params[cf.PARALLEL] = False params[cf.INTERFEROGRAM_FILES] = [ MultiplePaths(cls.BASE_OUT_DIR, p) for p in paths ] for p in params[cf.INTERFEROGRAM_FILES]: # cheat p.sampled_path = p.converted_path process.process_ifgs(sorted(paths), params, 2, 2) if not hasattr(cls, 'ifgs'): cls.ifgs = get_ifgs(out_dir=cls.BASE_OUT_DIR) except: # revert working dir & avoid paths busting other tests os.chdir(CURRENT_DIR) raise
def wrap_spatio_temporal_filter(params): """ A wrapper for the spatio-temporal filter so it can be tested. See docstring for spatio_temporal_filter. """ if params[cf.APSEST]: log.info('Doing APS spatio-temporal filtering') else: log.info('APS spatio-temporal filtering not required') return tiles = params[cf.TILES] preread_ifgs = params[cf.PREREAD_IFGS] ifg_paths = [ ifg_path.tmp_sampled_path for ifg_path in params[cf.INTERFEROGRAM_FILES] ] # perform some checks on existing ifgs log.debug('Checking APS correction status') if mpiops.run_once(shared.check_correction_status, ifg_paths, ifc.PYRATE_APS_ERROR): log.debug('Finished APS correction') return # return if True condition returned aps_error_files_on_disc = [ MultiplePaths.aps_error_path(i, params) for i in ifg_paths ] if all(a.exists() for a in aps_error_files_on_disc): log.warning("Reusing APS errors from previous run!!!") for ifg_path, a in mpiops.array_split( list(zip(ifg_paths, aps_error_files_on_disc))): phase = np.load(a) _save_aps_corrected_phase(ifg_path, phase) else: tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles) mpiops.comm.barrier() spatio_temporal_filter(tsincr, ifg_paths, params, preread_ifgs) mpiops.comm.barrier() shared.save_numpy_phase(ifg_paths, params)
def _make_aps_corrections(ts_aps: np.ndarray, ifgs: List[Ifg], params: dict) -> None: """ Function to convert the time series APS filter output into interferometric phase corrections and save them to disc. :param ts_aps: Incremental APS time series array. :param ifgs: List of Ifg class objects. :param params: Dictionary of PyRate configuration parameters. """ log.debug('Reconstructing interferometric observations from time series') # get first and second image indices _, n = mpiops.run_once(get_epochs, ifgs) index_first, index_second = n[:len(ifgs)], n[len(ifgs):] num_ifgs_tuples = mpiops.array_split(list(enumerate(ifgs))) for i, ifg in [(int(num), ifg) for num, ifg in num_ifgs_tuples]: # sum time slice data from first to second epoch ifg_aps = np.sum(ts_aps[:, :, index_first[i]:index_second[i]], axis=2) aps_error_on_disc = MultiplePaths.aps_error_path(ifg.tmp_path, params) np.save(file=aps_error_on_disc, arr=ifg_aps) # save APS as numpy array mpiops.comm.barrier()
def setup_class(cls): # change to orbital error correction method 2 cls.params = Configuration(common.TEST_CONF_ROIPAC).__dict__ cls.BASE_DIR = cls.params[cf.OUT_DIR] cls.params[cf.ORBITAL_FIT_METHOD] = NETWORK_METHOD cls.params[cf.ORBITAL_FIT_LOOKS_X] = 1 cls.params[cf.ORBITAL_FIT_LOOKS_Y] = 1 cls.params[cf.ORBFIT_OFFSET] = True cls.params[cf.OUT_DIR] = cls.BASE_DIR data_paths = [os.path.join(SML_TEST_TIF, p) for p in small_ifg_file_list()] cls.new_data_paths = [os.path.join(cls.BASE_DIR, os.path.basename(d)) for d in data_paths] cls.params[cf.INTERFEROGRAM_FILES] = [MultiplePaths(file_name=d, params=cls.params) for d in data_paths] for p in cls.params[cf.INTERFEROGRAM_FILES]: p.sampled_path = p.converted_path # copy the files from the dir into temp dir for d in data_paths: d_copy = os.path.join(cls.BASE_DIR, os.path.basename(d)) shutil.copy(d, d_copy) os.chmod(d_copy, 0o660) cls.headers = [roipac.roipac_header(i, cls.params) for i in cls.new_data_paths] cls.orb_error_dir = Path(cls.params[cf.OUT_DIR]).joinpath(ORB_ERROR_DIR) cls.orb_error_dir.mkdir(parents=True, exist_ok=True)
def independent_orbital_correction(ifg_path, params): """ Calculates and removes an orbital error surface from a single independent interferogram. Warning: This will write orbital error corrected phase_data to the ifg. :param Ifg class instance ifg: the interferogram to be corrected :param dict params: dictionary of configuration parameters :return: None - interferogram phase data is updated and saved to disk """ log.debug(f"Orbital correction of {ifg_path}") degree = params[C.ORBITAL_FIT_DEGREE] offset = params[C.ORBFIT_OFFSET] intercept = params[C.ORBFIT_INTERCEPT] xlooks = params[C.ORBITAL_FIT_LOOKS_X] ylooks = params[C.ORBITAL_FIT_LOOKS_Y] ifg0 = shared.Ifg(ifg_path) if isinstance(ifg_path, str) else ifg_path # get full-resolution design matrix fullres_dm = get_design_matrix(ifg0, degree, intercept=intercept) ifg = shared.dem_or_ifg(ifg_path) if isinstance(ifg_path, str) else ifg_path multi_path = MultiplePaths(ifg.data_path, params) orb_on_disc = MultiplePaths.orb_error_path(ifg.data_path, params) if not ifg.is_open: ifg.open() shared.nan_and_mm_convert(ifg, params) fullres_ifg = ifg # keep a backup fullres_phase = fullres_ifg.phase_data if orb_on_disc.exists(): log.info( f'Reusing already computed orbital fit correction: {orb_on_disc}') orbital_correction = np.load(file=orb_on_disc) else: # Multi-look the ifg data if either X or Y is greater than 1 if (xlooks > 1) or (ylooks > 1): exts, _, _ = __extents_from_params(params) mlooked = _create_mlooked_dataset(multi_path, ifg.data_path, exts, params) ifg = Ifg(mlooked) # multi-looked Ifg object ifg.initialize() shared.nan_and_mm_convert(ifg, params) # vectorise phase data, keeping NODATA vphase = reshape(ifg.phase_data, ifg.num_cells) # compute design matrix for multi-looked data mlooked_dm = get_design_matrix(ifg, degree, intercept=intercept) # invert to obtain the correction image (forward model) at full-res orbital_correction = __orb_correction(fullres_dm, mlooked_dm, fullres_phase, vphase, offset=offset) # save correction to disc if not orb_on_disc.parent.exists(): shared.mkdir_p(orb_on_disc.parent) np.save(file=orb_on_disc, arr=orbital_correction) # subtract orbital correction from the full-res ifg fullres_ifg.phase_data -= orbital_correction # set orbfit meta tag and save phase to file _save_orbital_error_corrected_phase(fullres_ifg, params)
def test_coherence_files_not_converted(): # define constants NO_DATA_VALUE = 0 driver = gdal.GetDriverByName('GTiff') # create a sample gdal dataset # sample gdal dataset sample_gdal_filename = "dataset_01122000.tif" options = ['PROFILE=GeoTIFF'] sample_gdal_dataset = driver.Create(sample_gdal_filename, 5, 5, 1, gdal.GDT_Float32, options=options) srs = osr.SpatialReference() wkt_projection = srs.ExportToWkt() sample_gdal_dataset.SetProjection(wkt_projection) sample_gdal_band = sample_gdal_dataset.GetRasterBand(1) sample_gdal_band.SetNoDataValue(NO_DATA_VALUE) sample_gdal_band.WriteArray(np.arange(25).reshape(5, 5)) sample_gdal_dataset.SetMetadataItem(ifc.FIRST_DATE, '2019-10-20') sample_gdal_dataset.SetMetadataItem(ifc.SECOND_DATE, '2019-11-01') sample_gdal_dataset.SetMetadataItem(ifc.PYRATE_WAVELENGTH_METRES, '10.05656') sample_gdal_dataset.FlushCache() sample_gdal_dataset = None ifg = Ifg(sample_gdal_filename) ifg.open() # create a coherence mask dataset tmpdir = tempfile.mkdtemp() out_dir = Path(tmpdir) # we won't be creating any output coherence mask files as there are already GeoTIFFs params = common.min_params(out_dir) coherence_mask_filename = MultiplePaths(Path("mask_dataset_01122000-02122000.tif").as_posix(), params) coherence_mask_dataset = driver.Create(coherence_mask_filename.converted_path, 5, 5, 1, gdal.GDT_Float32) srs = osr.SpatialReference() wkt_projection = srs.ExportToWkt() coherence_mask_dataset.SetProjection(wkt_projection) coherence_mask_band = coherence_mask_dataset.GetRasterBand(1) coherence_mask_band.SetNoDataValue(NO_DATA_VALUE) arr = np.arange(0, 75, 3).reshape(5, 5) / 100.0 arr[3, 4] = 0.25 # insert some random lower than threshold number arr[4, 2] = 0.20 # insert some random lower than threshold number coherence_mask_band.WriteArray(arr) # del the tmp handler datasets created del coherence_mask_dataset # create an artificial masked dataset expected_result_array = np.nan_to_num( np.array( [ [np.nan, np.nan, np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan, np.nan, np.nan], [10.0, 11.0, 12.0, 13.0, 14.0], [15.0, 16.0, 17.0, 18.0, np.nan], [20.0, 21.0, np.nan, 23.0, 24.0], ] ) ) # use the gdal_python.coherence_masking to find the actual mask dataset coherence_thresh = 0.3 gdal_python.coherence_masking(ifg.dataset, coherence_mask_filename.converted_path, coherence_thresh) sample_gdal_array = np.nan_to_num(ifg.phase_data) # compare the artificial masked and actual masked datasets np.testing.assert_array_equal(sample_gdal_array, expected_result_array) # del the tmp datasets created os.remove(coherence_mask_filename.converted_path) ifg.close() os.remove(sample_gdal_filename)
def mlooked_path(path, params, input_type): m = MultiplePaths(path, params=params, input_type=input_type) return m.sampled_path
def setUpClass(cls): rate_types = ['stack_rate', 'stack_error', 'stack_samples'] cls.tif_dir = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_GAMMA from pyrate.configuration import Configuration # change the required params params = Configuration(cls.test_conf).__dict__ params[cf.OBS_DIR] = common.SML_TEST_GAMMA params[cf.PROCESSES] = 4 params[cf.PROCESSOR] = 1 # gamma params[cf.IFG_FILE_LIST] = os.path.join(common.SML_TEST_GAMMA, 'ifms_17') params[cf.OUT_DIR] = cls.tif_dir params[cf.PARALLEL] = 1 params[cf.APS_CORRECTION] = False params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) rows, cols = params["rows"], params["cols"] # xlks, ylks, crop = cf.transform_params(params) # base_unw_paths need to be geotiffed by converttogeotif # and multilooked by run_prepifg base_unw_paths = list(cf.parse_namelist(params[cf.IFG_FILE_LIST])) multi_paths = [ MultiplePaths(params[cf.OUT_DIR], b, ifglksx=params[cf.IFG_LKSX], ifgcropopt=params[cf.IFG_CROP_OPT]) for b in base_unw_paths ] # dest_paths are tifs that have been geotif converted and multilooked cls.converted_paths = [b.converted_path for b in multi_paths] cls.sampled_paths = [b.sampled_path for b in multi_paths] from copy import copy orig_params = copy(params) conv2tif.main(params) prepifg.main(orig_params) tiles = pyrate.core.shared.get_tiles(cls.sampled_paths[0], rows, cols) ifgs = common.small_data_setup() params[cf.INTERFEROGRAM_FILES] = multi_paths cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = process.process_ifgs( cls.sampled_paths, params, rows, cols) cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) cls.rate_p, cls.error_p, cls.samples_p = \ [common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types] common.remove_tifs(params[cf.OBS_DIR]) # now create the non parallel version cls.tif_dir_s = tempfile.mkdtemp() params[cf.PARALLEL] = 0 params[cf.PROCESSES] = 1 params[cf.OUT_DIR] = cls.tif_dir_s params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) multi_paths = [ MultiplePaths(params[cf.OUT_DIR], b, ifglksx=params[cf.IFG_LKSX], ifgcropopt=params[cf.IFG_CROP_OPT]) for b in base_unw_paths ] cls.converted_paths_s = [b.converted_path for b in multi_paths] cls.sampled_paths_s = [b.sampled_path for b in multi_paths] orig_params = copy(params) conv2tif.main(params) prepifg.main(orig_params) params[cf.INTERFEROGRAM_FILES] = multi_paths cls.refpixel, cls.maxvar, cls.vcmt = process.process_ifgs( cls.sampled_paths_s, params, rows, cols) cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) cls.rate, cls.error, cls.samples = \ [common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types]