def _prepifg_multiprocessing(path, xlooks, ylooks, exts, thresh, crop, params): """ Multiprocessing wrapper for prepifg """ processor = params[cf.PROCESSOR] # roipac or gamma if processor == GAMMA: header = gamma.gamma_header(path, params) elif processor == ROIPAC: header = roipac.roipac_header(path, params) else: raise PreprocessError('Processor must be ROI_PAC (0) or GAMMA (1)') # If we're performing coherence masking, find the coherence file for this IFG. # TODO: Refactor _is_interferogram to be unprotected (remove '_') if params[cf.COH_MASK] and shared._is_interferogram(header): coherence_path = cf.coherence_paths_for(path, params, tif=True)[0] coherence_thresh = params[cf.COH_THRESH] else: coherence_path = None coherence_thresh = None prepifg_helper.prepare_ifg(path, xlooks, ylooks, exts, thresh, crop, out_path=params[cf.OUT_DIR], header=header, coherence_path=coherence_path, coherence_thresh=coherence_thresh)
def _geotiff_multiprocessing(unw_path, params): """ Multiprocessing wrapper for full-res geotiff conversion """ # TODO: Need a more robust method for identifying coherence files. if params[cf.COH_FILE_DIR] and unw_path.endswith('.cc'): # If the user has provided a dir for coherence files, place # converted coherence files in that directory. dest = shared.output_tiff_filename(unw_path, params[cf.COH_FILE_DIR]) else: dest = shared.output_tiff_filename(unw_path, params[cf.OBS_DIR]) processor = params[cf.PROCESSOR] # roipac or gamma # Create full-res geotiff if not already on disk if not os.path.exists(dest): if processor == GAMMA: header = gamma.gamma_header(unw_path, params) elif processor == ROIPAC: header = roipac.roipac_header(unw_path, params) else: raise PreprocessError('Processor must be ROI_PAC (0) or GAMMA (1)') shared.write_fullres_geotiff(header, unw_path, dest, nodata=params[cf.NO_DATA_VALUE]) return dest else: log.info("Full-res geotiff already exists") return None
def _prepifg_multiprocessing(path, xlooks, ylooks, exts, thresh, crop, params): """ Multiprocessing wrapper for prepifg """ processor = params[cf.PROCESSOR] # roipac, gamma or geotif if (processor == GAMMA) or (processor == GEOTIF): header = gamma.gamma_header(path, params) elif processor == ROIPAC: log.info("Warning: ROI_PAC support will be deprecated in a future PyRate release") header = roipac.roipac_header(path, params) else: raise PreprocessError('Processor must be ROI_PAC (0) or GAMMA (1)') # If we're performing coherence masking, find the coherence file for this IFG. if params[cf.COH_MASK] and shared._is_interferogram(header): coherence_path = cf.coherence_paths_for(path, params, tif=True) coherence_thresh = params[cf.COH_THRESH] else: coherence_path = None coherence_thresh = None if params[cf.LARGE_TIFS]: op = output_tiff_filename(path, params[cf.OUT_DIR]) looks_path = cf.mlooked_path(op, ylooks, crop) return path, coherence_path, looks_path else: prepifg_helper.prepare_ifg(path, xlooks, ylooks, exts, thresh, crop, out_path=params[cf.OUT_DIR], header=header, coherence_path=coherence_path, coherence_thresh=coherence_thresh)
def setUp(self): self.BASE_DIR = tempfile.mkdtemp() # change to orbital error correction method 2 self.params = Configuration(common.TEST_CONF_ROIPAC).__dict__ self.params[cf.ORBITAL_FIT_METHOD] = NETWORK_METHOD self.params[cf.ORBITAL_FIT_LOOKS_X] = 1 self.params[cf.ORBITAL_FIT_LOOKS_Y] = 1 data_paths = [ os.path.join(SML_TEST_TIF, p) for p in small_ifg_file_list() ] self.new_data_paths = [ os.path.join(self.BASE_DIR, os.path.basename(d)) for d in data_paths ] for d in data_paths: d_copy = os.path.join(self.BASE_DIR, os.path.basename(d)) shutil.copy(d, d_copy) os.chmod(d_copy, 0o660) self.ifgs = small_data_setup(datafiles=self.new_data_paths) self.headers = [ roipac.roipac_header(i.data_path, self.params) for i in self.ifgs ] for i in self.ifgs: if not i.is_open: i.open() if not i.nan_converted: i.convert_to_nans() if not i.mm_converted: i.convert_to_mm() i.write_modified_phase()
def _geotiff_multiprocessing(unw_path: MultiplePaths, params: dict) -> Tuple[str, bool]: """ Multiprocessing wrapper for full-res geotiff conversion """ # TODO: Need a more robust method for identifying coherence files. dest = unw_path.converted_path processor = params[C.PROCESSOR] # roipac or gamma # Create full-res geotiff if not already on disk if not os.path.exists(dest): if processor == GAMMA: header = gamma.gamma_header(unw_path.unwrapped_path, params) elif processor == ROIPAC: log.info( "Warning: ROI_PAC support will be deprecated in a future PyRate release" ) header = roipac.roipac_header(unw_path.unwrapped_path, params) else: raise PreprocessError('Processor must be ROI_PAC (0) or GAMMA (1)') header[ifc.INPUT_TYPE] = unw_path.input_type shared.write_fullres_geotiff(header, unw_path.unwrapped_path, dest, nodata=params[C.NO_DATA_VALUE]) Path(dest).chmod(0o444) # readonly output return dest, True else: log.warning( f"Full-res geotiff already exists in {dest}! Returning existing geotiff!" ) return dest, False
def setup_class(cls): cls.xs = 0.000833333 cls.ys = -cls.xs cls.ifgs, cls.random_dir = diff_exts_ifgs() cls.ifg_paths = [i.data_path for i in cls.ifgs] cls.params = Configuration(common.TEST_CONF_ROIPAC).__dict__ cls.params[C.OUT_DIR] = cls.random_dir cls.params[C.GEOMETRY_DIR] = Path(cls.random_dir).joinpath( C.GEOMETRY_DIR) cls.params[C.GEOMETRY_DIR].mkdir(exist_ok=True) cls.params[C.INTERFEROGRAM_DIR] = Path(cls.random_dir).joinpath( C.INTERFEROGRAM_DIR) cls.params[C.INTERFEROGRAM_DIR].mkdir(exist_ok=True) cls.headers = [ roipac.roipac_header(i.data_path, cls.params) for i in cls.ifgs ] paths = [ "060619-061002_ifg.tif", "060619-061002_ifg.tif", "060619-061002_ifg.tif", "060619-061002_ifg.tif", "070326-070917_ifg.tif", "070326-070917_ifg.tif", "070326-070917_ifg.tif", "070326-070917_ifg.tif" ] cls.exp_files = [ join(cls.random_dir, C.INTERFEROGRAM_DIR, p) for p in paths ]
def repair_params_for_correct_tests(out_dir, params): base_ifg_paths = [c.unwrapped_path for c in params[cf.INTERFEROGRAM_FILES]] headers = [roipac.roipac_header(i, params) for i in base_ifg_paths] params[cf.INTERFEROGRAM_FILES] = params[cf.INTERFEROGRAM_FILES][:-2] dest_paths = [ Path(out_dir).joinpath(Path(c.sampled_path).name).as_posix() for c in params[cf.INTERFEROGRAM_FILES] ] for p, d in zip(params[cf.INTERFEROGRAM_FILES], dest_paths): # hack p.sampled_path = d return dest_paths, headers
def find_header(path: MultiplePaths, params: dict): processor = params[cf.PROCESSOR] # roipac, gamma or geotif tif_path = path.converted_path if (processor == GAMMA) or (processor == GEOTIF): header = gamma.gamma_header(tif_path, params) elif processor == ROIPAC: log.info("Warning: ROI_PAC support will be deprecated in a future PyRate release") header = roipac.roipac_header(tif_path, params) else: raise PreprocessError('Processor must be ROI_PAC (0) or GAMMA (1)') header[ifc.INPUT_TYPE] = path.input_type return header
def setUpClass(cls): params = Configuration(common.TEST_CONF_ROIPAC).__dict__ shutil.rmtree(params[cf.OUT_DIR]) params = Configuration(common.TEST_CONF_ROIPAC).__dict__ cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['prepifg.py', common.TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir params[cf.TMPDIR] = cls.temp_out_dir conv2tif.main(params) prepifg.main(params) params[cf.REF_EST_METHOD] = 1 params[cf.PARALLEL] = True xlks, ylks, crop = cf.transform_params(params) base_ifg_paths = [ c.unwrapped_path for c in params[cf.INTERFEROGRAM_FILES] ] headers = [roipac.roipac_header(i, params) for i in base_ifg_paths] dest_paths = [ Path(cls.temp_out_dir).joinpath(Path( c.sampled_path).name).as_posix() for c in params[cf.INTERFEROGRAM_FILES][:-2] ] # start run_pyrate copy ifgs = common.pre_prepare_ifgs(dest_paths, params) mst_grid = common.mst_calculation(dest_paths, params) # Estimate reference pixel location refx, refy = process._ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.core.orbital.remove_orbital_error(ifgs, params, headers) for i in ifgs: i.close() ifgs = common.pre_prepare_ifgs(dest_paths, params) for i in ifgs: i.close() cls.ref_phs, cls.ifgs = process._ref_phase_estimation( dest_paths, params, refx, refy)
def setUpClass(cls): params = Configuration(TEST_CONF_ROIPAC).__dict__ cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['prepifg.py', TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir params[cf.TMPDIR] = os.path.join(cls.temp_out_dir, cf.TMPDIR) shared.mkdir_p(params[cf.TMPDIR]) params[cf.REF_EST_METHOD] = 2 conv2tif.main(params) prepifg.main(params) cls.params = params base_ifg_paths = [ c.unwrapped_path for c in params[cf.INTERFEROGRAM_FILES] ] dest_paths = [c.converted_path for c in params[cf.INTERFEROGRAM_FILES]] dest_paths = dest_paths[:-2] for i in dest_paths: Path(i).chmod( 0o664 ) # assign write permission as conv2tif output is readonly ifgs = common.pre_prepare_ifgs(dest_paths, params) refx, refy = process._ref_pixel_calc(dest_paths, params) headers = [roipac.roipac_header(i, cls.params) for i in base_ifg_paths] pyrate.core.orbital.remove_orbital_error(ifgs, params, headers) ifgs = prepare_ifgs_without_phase(dest_paths, params) for ifg in ifgs: ifg.close() _, cls.ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy) ifgs[0].open() r_dist = RDist(ifgs[0])() ifgs[0].close() # Calculate interferogram noise cls.maxvar = [ cvd(i, params, r_dist, calc_alpha=True, save_acg=True, write_vals=True)[0] for i in dest_paths ] cls.vcmt = get_vcmt(ifgs, cls.maxvar) for ifg in ifgs: ifg.close()
def setUp(self): from tests.common import small_data_setup self.ifgs = small_data_setup() self.ifg_paths = [i.data_path for i in self.ifgs] params = Configuration(common.TEST_CONF_ROIPAC).__dict__ self.headers = [ roipac.roipac_header(i.data_path, params) for i in self.ifgs ] prepare_ifgs(self.ifg_paths, crop_opt=1, xlooks=1, ylooks=1, headers=self.headers) looks_paths = [ mlooked_path(d, looks=1, crop_out=1) for d in self.ifg_paths ] self.ifgs_with_nan = [Ifg(i) for i in looks_paths] for ifg in self.ifgs_with_nan: ifg.open()
def setUp(self): self.xs = 0.000833333 self.ys = -self.xs self.ifgs, self.random_dir = diff_exts_ifgs() self.ifg_paths = [i.data_path for i in self.ifgs] params = Configuration(common.TEST_CONF_ROIPAC).__dict__ self.headers = [ roipac.roipac_header(i.data_path, params) for i in self.ifgs ] paths = [ "geo_060619-061002_unw_1rlks_1cr.tif", "geo_060619-061002_unw_1rlks_2cr.tif", "geo_060619-061002_unw_1rlks_3cr.tif", "geo_060619-061002_unw_4rlks_3cr.tif", "geo_070326-070917_unw_1rlks_1cr.tif", "geo_070326-070917_unw_1rlks_2cr.tif", "geo_070326-070917_unw_1rlks_3cr.tif", "geo_070326-070917_unw_4rlks_3cr.tif" ] self.exp_files = [join(self.random_dir, p) for p in paths]
def setup_class(cls): from tests.common import small_data_setup cls.ifgs = small_data_setup() cls.ifg_paths = [i.data_path for i in cls.ifgs] params = Configuration(common.TEST_CONF_ROIPAC).__dict__ cls.headers = [ roipac.roipac_header(i.data_path, params) for i in cls.ifgs ] params[C.IFG_LKSX], params[C.IFG_LKSY], params[ C.IFG_CROP_OPT] = 1, 1, 1 prepare_ifgs(cls.ifg_paths, crop_opt=1, xlooks=1, ylooks=1, headers=cls.headers, params=params) looks_paths = [ mlooked_path(d, params, t) for d, t in zip(cls.ifg_paths, [InputTypes.IFG] * len(cls.ifgs)) ] cls.ifgs_with_nan = [Ifg(i) for i in looks_paths] for ifg in cls.ifgs_with_nan: ifg.open()
def setup_class(cls): # change to orbital error correction method 2 cls.params = Configuration(common.TEST_CONF_ROIPAC).__dict__ cls.BASE_DIR = cls.params[cf.OUT_DIR] cls.params[cf.ORBITAL_FIT_METHOD] = NETWORK_METHOD cls.params[cf.ORBITAL_FIT_LOOKS_X] = 1 cls.params[cf.ORBITAL_FIT_LOOKS_Y] = 1 cls.params[cf.ORBFIT_OFFSET] = True cls.params[cf.OUT_DIR] = cls.BASE_DIR data_paths = [os.path.join(SML_TEST_TIF, p) for p in small_ifg_file_list()] cls.new_data_paths = [os.path.join(cls.BASE_DIR, os.path.basename(d)) for d in data_paths] cls.params[cf.INTERFEROGRAM_FILES] = [MultiplePaths(file_name=d, params=cls.params) for d in data_paths] for p in cls.params[cf.INTERFEROGRAM_FILES]: p.sampled_path = p.converted_path # copy the files from the dir into temp dir for d in data_paths: d_copy = os.path.join(cls.BASE_DIR, os.path.basename(d)) shutil.copy(d, d_copy) os.chmod(d_copy, 0o660) cls.headers = [roipac.roipac_header(i, cls.params) for i in cls.new_data_paths] cls.orb_error_dir = Path(cls.params[cf.OUT_DIR]).joinpath(ORB_ERROR_DIR) cls.orb_error_dir.mkdir(parents=True, exist_ok=True)
def setUpClass(cls): params = Configuration(common.TEST_CONF_ROIPAC).__dict__ cls.temp_out_dir = tempfile.mkdtemp() params[cf.OUT_DIR] = cls.temp_out_dir params[cf.PARALLEL] = 0 conv2tif.main(params) prepifg.main(params) params[cf.REF_EST_METHOD] = 2 base_ifg_paths = [ c.unwrapped_path for c in params[cf.INTERFEROGRAM_FILES] ] headers = [roipac.roipac_header(i, params) for i in base_ifg_paths] dest_paths = [ Path(cls.temp_out_dir).joinpath(Path( c.sampled_path).name).as_posix() for c in params[cf.INTERFEROGRAM_FILES][:-2] ] # start run_pyrate copy ifgs = common.pre_prepare_ifgs(dest_paths, params) mst_grid = common.mst_calculation(dest_paths, params) refx, refy = process._ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.core.orbital.remove_orbital_error(ifgs, params, headers) ifgs = common.prepare_ifgs_without_phase(dest_paths, params) for ifg in ifgs: ifg.close() _, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy) ifgs[0].open() r_dist = covariance.RDist(ifgs[0])() ifgs[0].close() # Calculate interferogram noise maxvar = [covariance.cvd(i, params, r_dist)[0] for i in dest_paths] for ifg in ifgs: ifg.open() vcmt = covariance.get_vcmt(ifgs, maxvar) for ifg in ifgs: ifg.close() ifg.open() ifg.nodata_value = 0.0 params[cf.TIME_SERIES_METHOD] = 2 params[cf.PARALLEL] = 1 # Calculate time series cls.tsincr, cls.tscum, _ = common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid) params[cf.PARALLEL] = 0 # Calculate time series serailly by the pixel cls.tsincr_0, cls.tscum_0, _ = common.calculate_time_series( ifgs, params, vcmt, mst=mst_grid) # copy legacy data SML_TIME_SERIES_DIR = os.path.join(common.SML_TEST_DIR, 'time_series') tsincr_path = os.path.join(SML_TIME_SERIES_DIR, 'ts_incr_interp0_method2.csv') ts_incr = np.genfromtxt(tsincr_path) tscum_path = os.path.join(SML_TIME_SERIES_DIR, 'ts_cum_interp0_method2.csv') ts_cum = np.genfromtxt(tscum_path) cls.ts_incr = np.reshape(ts_incr, newshape=cls.tsincr_0.shape, order='F') cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F')
def setUpClass(cls): params = Configuration(TEST_CONF_ROIPAC).__dict__ cls.temp_out_dir = tempfile.mkdtemp() params[cf.OUT_DIR] = cls.temp_out_dir params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) shared.mkdir_p(params[cf.TMPDIR]) conv2tif.main(params) prepifg.main(params) params[cf.REF_EST_METHOD] = 2 xlks, _, crop = cf.transform_params(params) base_ifg_paths = [ c.unwrapped_path for c in params[cf.INTERFEROGRAM_FILES] ] headers = [roipac.roipac_header(i, params) for i in base_ifg_paths] dest_paths = [ Path(cls.temp_out_dir).joinpath(Path( c.sampled_path).name).as_posix() for c in params[cf.INTERFEROGRAM_FILES][:-2] ] # start run_pyrate copy ifgs = pre_prepare_ifgs(dest_paths, params) mst_grid = tests.common.mst_calculation(dest_paths, params) refx, refy = process._ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.core.orbital.remove_orbital_error(ifgs, params, headers) ifgs = prepare_ifgs_without_phase(dest_paths, params) for ifg in ifgs: ifg.close() _, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy) ifgs[0].open() r_dist = vcm_module.RDist(ifgs[0])() ifgs[0].close() maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in dest_paths] for ifg in ifgs: ifg.open() vcmt = vcm_module.get_vcmt(ifgs, maxvar) for ifg in ifgs: ifg.close() ifg.open() # Calculate stacked rate map params[cf.PARALLEL] = 1 cls.rate, cls.error, cls.samples = tests.common.calculate_stack_rate( ifgs, params, vcmt, mst_mat=mst_grid) # Calculate stacked rate map params[cf.PARALLEL] = 0 cls.rate_s, cls.error_s, cls.samples_s = tests.common.calculate_stack_rate( ifgs, params, vcmt, mst_mat=mst_grid) stackrate_dir = os.path.join(SML_TEST_DIR, 'stackrate') cls.rate_container = np.genfromtxt(os.path.join( stackrate_dir, 'stackmap.csv'), delimiter=',') cls.error_container = np.genfromtxt(os.path.join( stackrate_dir, 'errormap.csv'), delimiter=',') cls.samples_container = np.genfromtxt(os.path.join( stackrate_dir, 'coh_sta.csv'), delimiter=',') for ifg in ifgs: ifg.close()