def setUpClass(cls): cls.tif_dir = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_GAMMA # change the required params cls.params = cf.get_config_params(cls.test_conf) cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA cls.params[cf.PROCESSOR] = 1 # gamma file_list = list( cf.parse_namelist(os.path.join(common.SML_TEST_GAMMA, 'ifms_17'))) fd, cls.params[cf.IFG_FILE_LIST] = tempfile.mkstemp(suffix='.conf', dir=cls.tif_dir) os.close(fd) # write a short filelist with only 3 gamma unws with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp: for f in file_list[:3]: fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n') cls.params[cf.OUT_DIR] = cls.tif_dir cls.params[cf.PARALLEL] = 0 cls.params[cf.REF_EST_METHOD] = 1 cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA # base_unw_paths need to be geotiffed and multilooked by run_prepifg cls.base_unw_paths = cf.original_ifg_paths( cls.params[cf.IFG_FILE_LIST], cls.params[cf.OBS_DIR]) cls.base_unw_paths.append(common.SML_TEST_DEM_GAMMA) xlks, ylks, crop = cf.transform_params(cls.params) # dest_paths are tifs that have been geotif converted and multilooked conv2tif.main(cls.params) prepifg.main(cls.params) # run_prepifg.gamma_prepifg(cls.base_unw_paths, cls.params) cls.base_unw_paths.pop() # removed dem as we don't want it in ifgs cls.dest_paths = cf.get_dest_paths(cls.base_unw_paths, crop, cls.params, xlks) cls.ifgs = common.small_data_setup(datafiles=cls.dest_paths)
def parallel_ifgs(gamma_conf): tdir = Path(tempfile.mkdtemp()) params_p = manipulate_test_conf(gamma_conf, tdir) params_p[cf.PARALLEL] = 1 output_conf_file = 'conf.conf' output_conf = tdir.joinpath(output_conf_file) cf.write_config_file(params=params_p, output_conf_file=output_conf) params_p = Configuration(output_conf).__dict__ gtif_paths = conv2tif.main(params_p) prepifg.main(params_p) parallel_df = list(Path(tdir).joinpath('out').glob(glob_prefix)) p_ifgs = small_data_setup(datafiles=parallel_df) yield p_ifgs shutil.rmtree(params_p[cf.OBS_DIR])
def test_prepifg_file_types(tempdir, gamma_conf, coh_mask): tdir = Path(tempdir()) params = manipulate_test_conf(gamma_conf, tdir) params[C.COH_MASK] = coh_mask params[C.PARALLEL] = 0 output_conf_file = 'conf.conf' output_conf = tdir.joinpath(output_conf_file) pyrate.configuration.write_config_file(params=params, output_conf_file=output_conf) params_s = Configuration(output_conf).__dict__ conv2tif.main(params_s) # reread params from config params_s = Configuration(output_conf).__dict__ prepifg.main(params_s) ifg_files = list( Path(tdir.joinpath(params_s[C.INTERFEROGRAM_DIR])).glob('*_unw.tif')) assert len(ifg_files) == 17 mlooked_files = list( Path(tdir.joinpath(params_s[C.INTERFEROGRAM_DIR])).glob('*_ifg.tif')) assert len(mlooked_files) == 17 coh_files = list( Path(tdir.joinpath(params_s[C.COHERENCE_DIR])).glob('*_cc.tif')) mlooked_coh_files = list( Path(tdir.joinpath(params_s[C.COHERENCE_DIR])).glob('*_coh.tif')) if coh_mask: assert len(coh_files) == 17 assert len(mlooked_coh_files) == 17 dem_file = list( Path(tdir.joinpath(params_s[C.GEOMETRY_DIR])).glob('*_dem.tif'))[0] mlooked_dem_file = list( Path(tdir.joinpath(params_s[C.GEOMETRY_DIR])).glob('dem.tif'))[0] import itertools # assert coherence and ifgs have correct metadata for i in itertools.chain( *[ifg_files, mlooked_files, coh_files, mlooked_coh_files]): ifg = Ifg(i) ifg.open() md = ifg.meta_data if i.name.endswith('_unw.tif'): assert md[ifc.DATA_TYPE] == ifc.ORIG assert ifc.IFG_LKSX not in md assert ifc.IFG_LKSY not in md assert ifc.IFG_CROP not in md continue if i.name.endswith('_cc.tif'): assert md[ifc.DATA_TYPE] == ifc.COH assert ifc.IFG_LKSX not in md assert ifc.IFG_LKSY not in md assert ifc.IFG_CROP not in md continue if i.name.endswith('_coh.tif'): assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED_COH assert md[ifc.IFG_LKSX] == '1' assert md[ifc.IFG_LKSY] == '1' assert md[ifc.IFG_CROP] == '1' continue if i.name.endswith('_ifg.tif'): if coh_mask: assert md[ifc.DATA_TYPE] == ifc.MLOOKED_COH_MASKED_IFG assert md[ifc.IFG_LKSX] == '1' assert md[ifc.IFG_LKSY] == '1' assert md[ifc.IFG_CROP] == '1' else: assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED assert md[ifc.IFG_LKSX] == '1' assert md[ifc.IFG_LKSY] == '1' assert md[ifc.IFG_CROP] == '1' continue # assert dem has correct metadata dem = DEM(dem_file.as_posix()) dem.open() md = dem.dataset.GetMetadata() assert md[ifc.DATA_TYPE] == ifc.DEM assert ifc.IFG_LKSX not in md assert ifc.IFG_LKSY not in md assert ifc.IFG_CROP not in md dem = DEM(mlooked_dem_file.as_posix()) dem.open() md = dem.dataset.GetMetadata() assert md[ifc.DATA_TYPE] == ifc.MLOOKED_DEM assert ifc.IFG_LKSX in md assert ifc.IFG_LKSY in md assert ifc.IFG_CROP in md shutil.rmtree(tdir)
def test_num_gamma_tifs_equals_num_unws(gamma_params): gtifs = conv2tif.main(gamma_params) # 17 unws + dem assert len(gtifs) == 18
def setup_class(cls): params = Configuration(common.TEST_CONF_ROIPAC).__dict__ params[C.TEMP_MLOOKED_DIR] = os.path.join(params[C.OUT_DIR], C.TEMP_MLOOKED_DIR) # force error maps to 1-sigma to match legacy params[C.VELERROR_NSIG] = 1 conv2tif.main(params) prepifg.main(params) params[C.REF_EST_METHOD] = 2 xlks, _, crop = pyrate.core.prepifg_helper.transform_params(params) dest_paths, headers = common.repair_params_for_correct_tests( params[C.INTERFEROGRAM_DIR], params) correct._copy_mlooked(params) copied_dest_paths = [ os.path.join(params[C.TEMP_MLOOKED_DIR], os.path.basename(d)) for d in dest_paths ] del dest_paths # start run_pyrate copy ifgs = pre_prepare_ifgs(copied_dest_paths, params) mst_grid = tests.common.mst_calculation(copied_dest_paths, params) refx, refy = pyrate.core.refpixel.ref_pixel_calc_wrapper(params) params[C.REFX] = refx params[C.REFY] = refy params[C.ORBFIT_OFFSET] = True # Estimate and remove orbit errors pyrate.core.orbital.remove_orbital_error(ifgs, params) ifgs = prepare_ifgs_without_phase(copied_dest_paths, params) for ifg in ifgs: ifg.close() correct._update_params_with_tiles(params) _, ifgs = pyrate.core.ref_phs_est.ref_phase_est_wrapper(params) ifgs[0].open() r_dist = vcm_module.RDist(ifgs[0])() ifgs[0].close() maxvar = [ vcm_module.cvd(i, params, r_dist)[0] for i in copied_dest_paths ] for ifg in ifgs: ifg.open() vcmt = vcm_module.get_vcmt(ifgs, maxvar) for ifg in ifgs: ifg.close() ifg.open() # Calculate stacked rate map params[C.PARALLEL] = 1 cls.rate, cls.error, cls.samples = tests.common.calculate_stack_rate( ifgs, params, vcmt, mst_mat=mst_grid) # Calculate stacked rate map params[C.PARALLEL] = 0 cls.rate_s, cls.error_s, cls.samples_s = tests.common.calculate_stack_rate( ifgs, params, vcmt, mst_mat=mst_grid) stackrate_dir = os.path.join(SML_TEST_DIR, 'stackrate') cls.rate_container = np.genfromtxt(os.path.join( stackrate_dir, 'stackmap.csv'), delimiter=',') cls.error_container = np.genfromtxt(os.path.join( stackrate_dir, 'errormap.csv'), delimiter=',') cls.samples_container = np.genfromtxt(os.path.join( stackrate_dir, 'coh_sta.csv'), delimiter=',') for ifg in ifgs: ifg.close() cls.params = params
def setUpClass(cls): rate_types = ['stack_rate', 'stack_error', 'stack_samples'] cls.tif_dir = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_GAMMA from pyrate.configuration import Configuration # change the required params params = Configuration(cls.test_conf).__dict__ params[cf.OBS_DIR] = common.SML_TEST_GAMMA params[cf.PROCESSES] = 4 params[cf.PROCESSOR] = 1 # gamma params[cf.IFG_FILE_LIST] = os.path.join(common.SML_TEST_GAMMA, 'ifms_17') params[cf.OUT_DIR] = cls.tif_dir params[cf.PARALLEL] = 1 params[cf.APS_CORRECTION] = False params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) rows, cols = params["rows"], params["cols"] # xlks, ylks, crop = cf.transform_params(params) # base_unw_paths need to be geotiffed by converttogeotif # and multilooked by run_prepifg base_unw_paths = list(cf.parse_namelist(params[cf.IFG_FILE_LIST])) multi_paths = [ MultiplePaths(params[cf.OUT_DIR], b, ifglksx=params[cf.IFG_LKSX], ifgcropopt=params[cf.IFG_CROP_OPT]) for b in base_unw_paths ] # dest_paths are tifs that have been geotif converted and multilooked cls.converted_paths = [b.converted_path for b in multi_paths] cls.sampled_paths = [b.sampled_path for b in multi_paths] from copy import copy orig_params = copy(params) conv2tif.main(params) prepifg.main(orig_params) tiles = pyrate.core.shared.get_tiles(cls.sampled_paths[0], rows, cols) ifgs = common.small_data_setup() params[cf.INTERFEROGRAM_FILES] = multi_paths cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = process.process_ifgs( cls.sampled_paths, params, rows, cols) cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) cls.rate_p, cls.error_p, cls.samples_p = \ [common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types] common.remove_tifs(params[cf.OBS_DIR]) # now create the non parallel version cls.tif_dir_s = tempfile.mkdtemp() params[cf.PARALLEL] = 0 params[cf.PROCESSES] = 1 params[cf.OUT_DIR] = cls.tif_dir_s params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) multi_paths = [ MultiplePaths(params[cf.OUT_DIR], b, ifglksx=params[cf.IFG_LKSX], ifgcropopt=params[cf.IFG_CROP_OPT]) for b in base_unw_paths ] cls.converted_paths_s = [b.converted_path for b in multi_paths] cls.sampled_paths_s = [b.sampled_path for b in multi_paths] orig_params = copy(params) conv2tif.main(params) prepifg.main(orig_params) params[cf.INTERFEROGRAM_FILES] = multi_paths cls.refpixel, cls.maxvar, cls.vcmt = process.process_ifgs( cls.sampled_paths_s, params, rows, cols) cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) cls.rate, cls.error, cls.samples = \ [common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types]
def setUpClass(cls): from pyrate import conv2tif, prepifg cls.params = config.get_config_params(TEST_CONF_GAMMA) conv2tif.main(cls.params) prepifg.main(cls.params)
def setup_class(cls): params = Configuration(common.TEST_CONF_ROIPAC).__dict__ params[cf.TEMP_MLOOKED_DIR] = os.path.join(params[cf.OUT_DIR], cf.TEMP_MLOOKED_DIR) conv2tif.main(params) prepifg.main(params) params[cf.REF_EST_METHOD] = 2 xlks, _, crop = cf.transform_params(params) dest_paths, headers = common.repair_params_for_correct_tests( params[cf.OUT_DIR], params) correct._copy_mlooked(params) copied_dest_paths = [ os.path.join(params[cf.TEMP_MLOOKED_DIR], os.path.basename(d)) for d in dest_paths ] del dest_paths # start run_pyrate copy ifgs = common.pre_prepare_ifgs(copied_dest_paths, params) mst_grid = common.mst_calculation(copied_dest_paths, params) refx, refy = pyrate.core.refpixel.ref_pixel_calc_wrapper(params) params[cf.REFX] = refx params[cf.REFY] = refy params[cf.ORBFIT_OFFSET] = True # Estimate and remove orbit errors pyrate.core.orbital.remove_orbital_error(ifgs, params) ifgs = common.prepare_ifgs_without_phase(copied_dest_paths, params) for ifg in ifgs: ifg.close() correct._update_params_with_tiles(params) _, ifgs = pyrate.core.ref_phs_est.ref_phase_est_wrapper(params) ifgs[0].open() r_dist = covariance.RDist(ifgs[0])() ifgs[0].close() # Calculate interferogram noise maxvar = [ covariance.cvd(i, params, r_dist)[0] for i in copied_dest_paths ] for ifg in ifgs: ifg.open() vcmt = covariance.get_vcmt(ifgs, maxvar) for ifg in ifgs: ifg.close() ifg.open() ifg.nodata_value = 0.0 params[cf.TIME_SERIES_METHOD] = 2 params[cf.PARALLEL] = 1 # Calculate time series cls.tsincr, cls.tscum, _ = common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid) params[cf.PARALLEL] = 0 # Calculate time series serailly by the pixel cls.tsincr_0, cls.tscum_0, _ = common.calculate_time_series( ifgs, params, vcmt, mst=mst_grid) # copy legacy data SML_TIME_SERIES_DIR = os.path.join(common.SML_TEST_DIR, 'time_series') tsincr_path = os.path.join(SML_TIME_SERIES_DIR, 'ts_incr_interp0_method2.csv') ts_incr = np.genfromtxt(tsincr_path) tscum_path = os.path.join(SML_TIME_SERIES_DIR, 'ts_cum_interp0_method2.csv') ts_cum = np.genfromtxt(tscum_path) cls.ts_incr = np.reshape(ts_incr, newshape=cls.tsincr_0.shape, order='F') cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F') cls.params = params
def setUpClass(cls): params = cf.get_config_params(common.TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['prepifg.py', common.TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir conv2tif.main(params) prepifg.main(params) params[cf.REF_EST_METHOD] = 2 xlks, ylks, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST], params[cf.OBS_DIR]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) # start run_pyrate copy ifgs = common.pre_prepare_ifgs(dest_paths, params) mst_grid = common.mst_calculation(dest_paths, params) refx, refy = process._ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.core.orbital.remove_orbital_error(ifgs, params) ifgs = common.prepare_ifgs_without_phase(dest_paths, params) for ifg in ifgs: ifg.close() _, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy) ifgs[0].open() r_dist = covariance.RDist(ifgs[0])() ifgs[0].close() # Calculate interferogram noise maxvar = [covariance.cvd(i, params, r_dist)[0] for i in dest_paths] for ifg in ifgs: ifg.open() vcmt = covariance.get_vcmt(ifgs, maxvar) for ifg in ifgs: ifg.close() ifg.open() ifg.nodata_value = 0.0 params[cf.TIME_SERIES_METHOD] = 2 params[cf.PARALLEL] = 1 # Calculate time series cls.tsincr, cls.tscum, _ = common.calculate_time_series( ifgs, params, vcmt, mst=mst_grid) params[cf.PARALLEL] = 2 # Calculate time series cls.tsincr_2, cls.tscum_2, _ = \ common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid) params[cf.PARALLEL] = 0 # Calculate time series serailly by the pixel cls.tsincr_0, cls.tscum_0, _ = \ common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid) # copy legacy data SML_TIME_SERIES_DIR = os.path.join(common.SML_TEST_DIR, 'time_series') tsincr_path = os.path.join(SML_TIME_SERIES_DIR, 'ts_incr_interp0_method2.csv') ts_incr = np.genfromtxt(tsincr_path) tscum_path = os.path.join(SML_TIME_SERIES_DIR, 'ts_cum_interp0_method2.csv') ts_cum = np.genfromtxt(tscum_path) cls.ts_incr = np.reshape(ts_incr, newshape=cls.tsincr_0.shape, order='F') cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F')
def setUpClass(cls): from pyrate import conv2tif, prepifg cls.params = Configuration(TEST_CONF_GAMMA).__dict__ conv2tif.main(cls.params) prepifg.main(cls.params)
def setUpClass(cls): params = cf.get_config_params(TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() params[cf.OUT_DIR] = cls.temp_out_dir params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) shared.mkdir_p(params[cf.TMPDIR]) conv2tif.main(params) prepifg.main(params) params[cf.REF_EST_METHOD] = 2 xlks, _, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST], params[cf.OBS_DIR]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) print(f"base_ifg_paths={base_ifg_paths}") print(f"dest_paths={dest_paths}") # start run_pyrate copy ifgs = pre_prepare_ifgs(dest_paths, params) mst_grid = tests.common.mst_calculation(dest_paths, params) refx, refy = process._ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.core.orbital.remove_orbital_error(ifgs, params) ifgs = prepare_ifgs_without_phase(dest_paths, params) for ifg in ifgs: ifg.close() _, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy) ifgs[0].open() r_dist = vcm_module.RDist(ifgs[0])() ifgs[0].close() maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in dest_paths] for ifg in ifgs: ifg.open() vcmt = vcm_module.get_vcmt(ifgs, maxvar) for ifg in ifgs: ifg.close() ifg.open() # Calculate linear rate map params[cf.PARALLEL] = 1 cls.rate, cls.error, cls.samples = tests.common.calculate_linear_rate( ifgs, params, vcmt, mst_mat=mst_grid) params[cf.PARALLEL] = 2 cls.rate_2, cls.error_2, cls.samples_2 = \ tests.common.calculate_linear_rate(ifgs, params, vcmt, mst_mat=mst_grid) params[cf.PARALLEL] = 0 # Calculate linear rate map cls.rate_s, cls.error_s, cls.samples_s = \ tests.common.calculate_linear_rate(ifgs, params, vcmt, mst_mat=mst_grid) linrate_dir = os.path.join(SML_TEST_DIR, 'linrate') cls.rate_container = np.genfromtxt(os.path.join( linrate_dir, 'stackmap.csv'), delimiter=',') cls.error_container = np.genfromtxt(os.path.join( linrate_dir, 'errormap.csv'), delimiter=',') cls.samples_container = np.genfromtxt(os.path.join( linrate_dir, 'coh_sta.csv'), delimiter=',') for ifg in ifgs: ifg.close()
def test_tifs_succeeds(self): conv2tif.main(self.params) prepifg.main(self.params)
def test_num_roipac_tifs_equals_num_unws(self): gtifs = conv2tif.main(self.roipac_params) # 17 unws + dem self.assertEqual(len(gtifs), 18)
def test_num_gamma_tifs_equals_num_unws(self): gtifs = conv2tif.main(self.gamma_params) # 17 unws + incidence + dem self.assertEqual(len(gtifs), 19)
def test_tifs_succeeds(gamma_params): conv2tif.main(gamma_params) prepifg.main(gamma_params)
def test_num_roipac_tifs_equals_num_unws(roipac_params): gtifs = conv2tif.main(roipac_params) # 17 unws + dem assert len(gtifs) == 18
def main(): start_time = time.time() parser = argparse.ArgumentParser(prog='pyrate', description=CLI_DESCRIPTION, add_help=True, formatter_class=RawTextHelpFormatter) parser.add_argument('-v', '--verbosity', type=str, default='INFO', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'], help="Increase output verbosity") subparsers = parser.add_subparsers(dest='command') subparsers.required = True parser_conv2tif = subparsers.add_parser('conv2tif', help='Convert interferograms to geotiff.', add_help=True) parser_conv2tif.add_argument('-f', '--config_file', action="store", type=str, default=None, help="Pass configuration file", required=True) parser_prepifg = subparsers.add_parser('prepifg', help='Perform multilooking and cropping on geotiffs.', add_help=True) parser_prepifg.add_argument('-f', '--config_file', action="store", type=str, default=None, help="Pass configuration file", required=True) parser_process = subparsers.add_parser( 'process', help='Main processing workflow including corrections, time series and stacking computation.', add_help=True) parser_process.add_argument('-f', '--config_file', action="store", type=str, default=None, help="Pass configuration file", required=True) parser_merge = subparsers.add_parser('merge', help="Reassemble computed tiles and save as geotiffs.", add_help=True) parser_merge.add_argument('-f', '--config_file', action="store", type=str, default=None, help="Pass configuration file", required=False) parser_workflow = subparsers.add_parser('workflow', help="Run all the PyRate processes", add_help=True) parser_workflow.add_argument('-f', '--config_file', action="store", type=str, default=None, help="Pass configuration file", required=False) args = parser.parse_args() params = mpiops.run_once(_params_from_conf, args.config_file) configure_stage_log(args.verbosity, args.command, Path(params[cf.OUT_DIR]).joinpath('pyrate.log.').as_posix()) log.debug("Starting PyRate") log.debug("Arguments supplied at command line: ") log.debug(args) if args.verbosity: log.setLevel(args.verbosity) log.info("Verbosity set to " + str(args.verbosity) + ".") if args.command == "conv2tif": conv2tif.main(params) if args.command == "prepifg": prepifg.main(params) if args.command == "process": process.main(params) if args.command == "merge": merge.main(params) if args.command == "workflow": log.info("***********CONV2TIF**************") conv2tif.main(params) log.info("***********PREPIFG**************") params = mpiops.run_once(_params_from_conf, args.config_file) prepifg.main(params) log.info("***********PROCESS**************") # reset params as prepifg modifies params params = mpiops.run_once(_params_from_conf, args.config_file) process.main(params) # process might modify params too params = mpiops.run_once(_params_from_conf, args.config_file) log.info("***********MERGE**************") merge.main(params) log.debug("--- %s seconds ---" % (time.time() - start_time))
def test_prepifg_file_types(tempdir, gamma_conf, coh_mask): tdir = Path(tempdir()) params = manipulate_test_conf(gamma_conf, tdir) params[cf.COH_MASK] = coh_mask params[cf.PARALLEL] = 0 output_conf_file = 'conf.conf' output_conf = tdir.joinpath(output_conf_file) cf.write_config_file(params=params, output_conf_file=output_conf) params_s = Configuration(output_conf).__dict__ conv2tif.main(params_s) # reread params from config params_s = Configuration(output_conf).__dict__ prepifg.main(params_s) ifg_files = list( Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_ifg.tif')) assert len(ifg_files) == 17 mlooked_files = list( Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_ifg_1rlks_1cr.tif')) assert len(mlooked_files) == 17 coh_files = list( Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_cc_coh.tif')) mlooked_coh_files = list( Path(tdir.joinpath( params_s[cf.OUT_DIR])).glob('*_cc_coh_1rlks_1cr.tif')) if coh_mask: assert len(coh_files) == 17 assert len(mlooked_coh_files) == 17 dem_file = list( Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_dem.tif'))[0] mlooked_dem_file = list( Path(tdir.joinpath( params_s[cf.OUT_DIR])).glob('*_dem_1rlks_1cr.tif'))[0] import itertools # assert coherence and ifgs have correct metadata for i in itertools.chain( *[ifg_files, mlooked_files, coh_files, mlooked_coh_files]): ifg = Ifg(i) ifg.open() md = ifg.meta_data if i.name.endswith('_ifg.tif'): assert md[ifc.DATA_TYPE] == ifc.ORIG continue if i.name.endswith('_coh.tif'): assert md[ifc.DATA_TYPE] == ifc.COH continue if i.name.endswith('_cc_coh_1rlks_1cr.tif'): assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED_COH continue if i.name.endswith('_ifg_1rlks_1cr.tif'): if coh_mask: assert md[ifc.DATA_TYPE] == ifc.COHERENCE else: assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED continue # assert dem has correct metadata dem = DEM(dem_file.as_posix()) dem.open() md = dem.dataset.GetMetadata() assert md[ifc.DATA_TYPE] == ifc.DEM dem = DEM(mlooked_dem_file.as_posix()) dem.open() md = dem.dataset.GetMetadata() assert md[ifc.DATA_TYPE] == ifc.MLOOKED_DEM shutil.rmtree(tdir)
def main(): start_time = time.time() parser = argparse.ArgumentParser(prog='pyrate', description=CLI_DESCRIPTION, add_help=True, formatter_class=RawTextHelpFormatter) parser.add_argument('-v', '--verbosity', type=str, default='INFO', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'], help="Increase output verbosity") subparsers = parser.add_subparsers(dest='command') subparsers.required = True parser_conv2tif = subparsers.add_parser('conv2tif', help='<Optional> Convert interferograms to geotiff.', add_help=True) parser_prepifg = subparsers.add_parser( 'prepifg', help='Perform multilooking, cropping and coherence masking to interferogram geotiffs.', add_help=True) parser_correct = subparsers.add_parser( 'correct', help='Calculate and apply corrections to interferogram phase data.', add_help=True) parser_ts = subparsers.add_parser( 'timeseries', help='<Optional> Timeseries inversion of interferogram phase data.', add_help=True ) parser_stack = subparsers.add_parser('stack', help='<Optional> Stacking of interferogram phase data.', add_help=True) parser_merge = subparsers.add_parser( 'merge', help="Reassemble computed tiles and save as geotiffs.", add_help=True) parser_workflow = subparsers.add_parser( 'workflow', help="<Optional> Sequentially run all the PyRate processing steps.", add_help=True) for p in [parser_conv2tif, parser_prepifg, parser_correct, parser_merge, parser_ts, parser_stack, parser_workflow]: p.add_argument('-f', '--config_file', action="store", type=str, default=None, help="Pass configuration file", required=False) args = parser.parse_args() params = mpiops.run_once(_params_from_conf, args.config_file) configure_stage_log(args.verbosity, args.command, Path(params[C.OUT_DIR]).joinpath('pyrate.log.').as_posix()) log.debug("Starting PyRate") log.debug("Arguments supplied at command line: ") log.debug(args) if args.verbosity: log.setLevel(args.verbosity) log.info("Verbosity set to " + str(args.verbosity) + ".") if args.command == "conv2tif": conv2tif.main(params) if args.command == "prepifg": prepifg.main(params) if args.command == "correct": config_file = os.path.abspath(args.config_file) config = Configuration(config_file) correct.main(config) if args.command == "timeseries": config_file = os.path.abspath(args.config_file) config = Configuration(config_file) timeseries(config) if args.command == "stack": config_file = os.path.abspath(args.config_file) config = Configuration(config_file) stack(config) if args.command == "merge": merge.main(params) if args.command == "workflow": log.info("***********CONV2TIF**************") conv2tif.main(params) log.info("***********PREPIFG**************") params = mpiops.run_once(_params_from_conf, args.config_file) prepifg.main(params) log.info("***********CORRECT**************") # reset params as prepifg modifies params config_file = os.path.abspath(args.config_file) config = Configuration(config_file) correct.main(config) log.info("***********TIMESERIES**************") config = Configuration(config_file) timeseries(config) log.info("***********STACK**************") config = Configuration(config_file) stack(config) log.info("***********MERGE**************") params = mpiops.run_once(_params_from_conf, args.config_file) merge.main(params) log.info("--- Runtime = %s seconds ---" % (time.time() - start_time))
def setUpClass(cls): params = Configuration(common.TEST_CONF_ROIPAC).__dict__ cls.temp_out_dir = tempfile.mkdtemp() params[cf.OUT_DIR] = cls.temp_out_dir params[cf.PARALLEL] = 0 conv2tif.main(params) prepifg.main(params) params[cf.REF_EST_METHOD] = 2 base_ifg_paths = [ c.unwrapped_path for c in params[cf.INTERFEROGRAM_FILES] ] headers = [roipac.roipac_header(i, params) for i in base_ifg_paths] dest_paths = [ Path(cls.temp_out_dir).joinpath(Path( c.sampled_path).name).as_posix() for c in params[cf.INTERFEROGRAM_FILES][:-2] ] # start run_pyrate copy ifgs = common.pre_prepare_ifgs(dest_paths, params) mst_grid = common.mst_calculation(dest_paths, params) refx, refy = process._ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.core.orbital.remove_orbital_error(ifgs, params, headers) ifgs = common.prepare_ifgs_without_phase(dest_paths, params) for ifg in ifgs: ifg.close() _, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy) ifgs[0].open() r_dist = covariance.RDist(ifgs[0])() ifgs[0].close() # Calculate interferogram noise maxvar = [covariance.cvd(i, params, r_dist)[0] for i in dest_paths] for ifg in ifgs: ifg.open() vcmt = covariance.get_vcmt(ifgs, maxvar) for ifg in ifgs: ifg.close() ifg.open() ifg.nodata_value = 0.0 params[cf.TIME_SERIES_METHOD] = 2 params[cf.PARALLEL] = 1 # Calculate time series cls.tsincr, cls.tscum, _ = common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid) params[cf.PARALLEL] = 0 # Calculate time series serailly by the pixel cls.tsincr_0, cls.tscum_0, _ = common.calculate_time_series( ifgs, params, vcmt, mst=mst_grid) # copy legacy data SML_TIME_SERIES_DIR = os.path.join(common.SML_TEST_DIR, 'time_series') tsincr_path = os.path.join(SML_TIME_SERIES_DIR, 'ts_incr_interp0_method2.csv') ts_incr = np.genfromtxt(tsincr_path) tscum_path = os.path.join(SML_TIME_SERIES_DIR, 'ts_cum_interp0_method2.csv') ts_cum = np.genfromtxt(tscum_path) cls.ts_incr = np.reshape(ts_incr, newshape=cls.tsincr_0.shape, order='F') cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F')
def setUpClass(cls): params = Configuration(TEST_CONF_ROIPAC).__dict__ cls.temp_out_dir = tempfile.mkdtemp() params[cf.OUT_DIR] = cls.temp_out_dir params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) shared.mkdir_p(params[cf.TMPDIR]) conv2tif.main(params) prepifg.main(params) params[cf.REF_EST_METHOD] = 2 xlks, _, crop = cf.transform_params(params) base_ifg_paths = [ c.unwrapped_path for c in params[cf.INTERFEROGRAM_FILES] ] headers = [roipac.roipac_header(i, params) for i in base_ifg_paths] dest_paths = [ Path(cls.temp_out_dir).joinpath(Path( c.sampled_path).name).as_posix() for c in params[cf.INTERFEROGRAM_FILES][:-2] ] # start run_pyrate copy ifgs = pre_prepare_ifgs(dest_paths, params) mst_grid = tests.common.mst_calculation(dest_paths, params) refx, refy = process._ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.core.orbital.remove_orbital_error(ifgs, params, headers) ifgs = prepare_ifgs_without_phase(dest_paths, params) for ifg in ifgs: ifg.close() _, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy) ifgs[0].open() r_dist = vcm_module.RDist(ifgs[0])() ifgs[0].close() maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in dest_paths] for ifg in ifgs: ifg.open() vcmt = vcm_module.get_vcmt(ifgs, maxvar) for ifg in ifgs: ifg.close() ifg.open() # Calculate stacked rate map params[cf.PARALLEL] = 1 cls.rate, cls.error, cls.samples = tests.common.calculate_stack_rate( ifgs, params, vcmt, mst_mat=mst_grid) # Calculate stacked rate map params[cf.PARALLEL] = 0 cls.rate_s, cls.error_s, cls.samples_s = tests.common.calculate_stack_rate( ifgs, params, vcmt, mst_mat=mst_grid) stackrate_dir = os.path.join(SML_TEST_DIR, 'stackrate') cls.rate_container = np.genfromtxt(os.path.join( stackrate_dir, 'stackmap.csv'), delimiter=',') cls.error_container = np.genfromtxt(os.path.join( stackrate_dir, 'errormap.csv'), delimiter=',') cls.samples_container = np.genfromtxt(os.path.join( stackrate_dir, 'coh_sta.csv'), delimiter=',') for ifg in ifgs: ifg.close()