def setUpClass(cls): params = Configuration(TEST_CONF_ROIPAC).__dict__ cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['prepifg.py', TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir params[cf.TMPDIR] = os.path.join(cls.temp_out_dir, cf.TMPDIR) shared.mkdir_p(params[cf.TMPDIR]) params[cf.REF_EST_METHOD] = 2 conv2tif.main(params) prepifg.main(params) cls.params = params xlks, ylks, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST], params[cf.OBS_DIR]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) ifgs = common.pre_prepare_ifgs(dest_paths, params) refx, refy = process._ref_pixel_calc(dest_paths, params) pyrate.core.orbital.remove_orbital_error(ifgs, params) ifgs = prepare_ifgs_without_phase(dest_paths, params) for ifg in ifgs: ifg.close() _, cls.ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy) ifgs[0].open() r_dist = RDist(ifgs[0])() ifgs[0].close() # Calculate interferogram noise cls.maxvar = [cvd(i, params, r_dist, calc_alpha=True, save_acg=True, write_vals=True)[0] for i in dest_paths] cls.vcmt = get_vcmt(ifgs, cls.maxvar) for ifg in ifgs: ifg.close()
def setUpClass(cls): params = Configuration(common.TEST_CONF_ROIPAC).__dict__ cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['prepifg.py', common.TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir conv2tif.main(params) prepifg.main(params) params[cf.REF_EST_METHOD] = 2 xlks, ylks, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST], params[cf.OBS_DIR]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) # start run_pyrate copy ifgs = common.pre_prepare_ifgs(dest_paths, params) mst_grid = common.mst_calculation(dest_paths, params) refx, refy = process._ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.core.orbital.remove_orbital_error(ifgs, params) ifgs = common.prepare_ifgs_without_phase(dest_paths, params) for ifg in ifgs: ifg.close() _, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy) ifgs[0].open() r_dist = covariance.RDist(ifgs[0])() ifgs[0].close() maxvar = [covariance.cvd(i, params, r_dist)[0] for i in dest_paths] for ifg in ifgs: ifg.open() vcmt = covariance.get_vcmt(ifgs, maxvar) for ifg in ifgs: ifg.close() ifg.open() ifg.nodata_value = 0.0 params[cf.TIME_SERIES_METHOD] = 1 params[cf.PARALLEL] = 0 # Calculate time series cls.tsincr_0, cls.tscum_0, _ = common.calculate_time_series( ifgs, params, vcmt, mst=mst_grid) params[cf.PARALLEL] = 1 cls.tsincr_1, cls.tscum_1, cls.tsvel_1 = common.calculate_time_series( ifgs, params, vcmt, mst=mst_grid) # load the legacy data ts_dir = os.path.join(common.SML_TEST_DIR, 'time_series') tsincr_path = os.path.join(ts_dir, 'ts_incr_interp0_method1.csv') ts_incr = np.genfromtxt(tsincr_path) tscum_path = os.path.join(ts_dir, 'ts_cum_interp0_method1.csv') ts_cum = np.genfromtxt(tscum_path) cls.ts_incr = np.reshape(ts_incr, newshape=cls.tsincr_0.shape, order='F') cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F')
def setup_class(cls): params = Configuration(common.TEST_CONF_ROIPAC).__dict__ params[cf.TEMP_MLOOKED_DIR] = os.path.join(params[cf.OUT_DIR], cf.TEMP_MLOOKED_DIR) conv2tif.main(params) prepifg.main(params) params[cf.REF_EST_METHOD] = 2 xlks, _, crop = cf.transform_params(params) dest_paths, headers = common.repair_params_for_correct_tests(params[cf.OUT_DIR], params) correct._copy_mlooked(params) copied_dest_paths = [os.path.join(params[cf.TEMP_MLOOKED_DIR], os.path.basename(d)) for d in dest_paths] del dest_paths # start run_pyrate copy ifgs = pre_prepare_ifgs(copied_dest_paths, params) mst_grid = tests.common.mst_calculation(copied_dest_paths, params) refx, refy = pyrate.core.refpixel.ref_pixel_calc_wrapper(params) params[cf.REFX] = refx params[cf.REFY] = refy params[cf.ORBFIT_OFFSET] = True # Estimate and remove orbit errors pyrate.core.orbital.remove_orbital_error(ifgs, params) ifgs = prepare_ifgs_without_phase(copied_dest_paths, params) for ifg in ifgs: ifg.close() correct._update_params_with_tiles(params) _, ifgs = pyrate.core.ref_phs_est.ref_phase_est_wrapper(params) ifgs[0].open() r_dist = vcm_module.RDist(ifgs[0])() ifgs[0].close() maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in copied_dest_paths] for ifg in ifgs: ifg.open() vcmt = vcm_module.get_vcmt(ifgs, maxvar) for ifg in ifgs: ifg.close() ifg.open() # Calculate stacked rate map params[cf.PARALLEL] = 1 cls.rate, cls.error, cls.samples = tests.common.calculate_stack_rate(ifgs, params, vcmt, mst_mat=mst_grid) # Calculate stacked rate map params[cf.PARALLEL] = 0 cls.rate_s, cls.error_s, cls.samples_s = tests.common.calculate_stack_rate(ifgs, params, vcmt, mst_mat=mst_grid) stackrate_dir = os.path.join(SML_TEST_DIR, 'stackrate') cls.rate_container = np.genfromtxt(os.path.join(stackrate_dir, 'stackmap.csv'), delimiter=',') cls.error_container = np.genfromtxt(os.path.join(stackrate_dir, 'errormap.csv'), delimiter=',') cls.samples_container = np.genfromtxt(os.path.join(stackrate_dir, 'coh_sta.csv'), delimiter=',') for ifg in ifgs: ifg.close() cls.params = params
def _merge_setup(rows, cols, params): """ Convenience function for Merge set up steps """ # setup paths xlks, _, crop = cf.transform_params(params) base_unw_paths = [] for p in Path(params[cf.OUT_DIR]).rglob("*rlks_*cr.tif"): if "dem" not in str(p): base_unw_paths.append(str(p)) if "tif" in base_unw_paths[0].split(".")[1]: dest_tifs = base_unw_paths # cf.get_dest_paths(base_unw_paths, crop, params, xlks) for i, dest_tif in enumerate(dest_tifs): dest_tifs[i] = dest_tif.replace("_tif", "") else: dest_tifs = base_unw_paths # cf.get_dest_paths(base_unw_paths, crop, params, xlks) # load previously saved preread_ifgs dict preread_ifgs_file = join(params[cf.TMPDIR], 'preread_ifgs.pk') ifgs_dict = pickle.load(open(preread_ifgs_file, 'rb')) ifgs = [v for v in ifgs_dict.values() if isinstance(v, shared.PrereadIfg)] shape = ifgs[0].shape tiles = shared.get_tiles(dest_tifs[0], rows, cols) return shape, tiles, ifgs_dict
def _merge_linrate(rows, cols, params): """ Merge linear rate outputs """ # pylint: disable=expression-not-assigned # setup paths xlks, _, crop = cf.transform_params(params) base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST], params[cf.OBS_DIR]) if "tif" in base_unw_paths[0].split(".")[1]: dest_tifs = cf.get_dest_paths(base_unw_paths, crop, params, xlks) for i, dest_tif in enumerate(dest_tifs): dest_tifs[i] = dest_tif.replace("_tif", "") else: dest_tifs = cf.get_dest_paths(base_unw_paths, crop, params, xlks) # load previously saved prepread_ifgs dict preread_ifgs_file = join(params[cf.TMPDIR], 'preread_ifgs.pk') ifgs = cp.load(open(preread_ifgs_file, 'rb')) tiles = shared.get_tiles(dest_tifs[0], rows, cols) # linrate aggregation if mpiops.size >= 3: [ _save_linrate(ifgs, params, tiles, out_type=t) for i, t in enumerate(['linrate', 'linerror', 'linsamples']) if i == mpiops.rank ] else: if mpiops.rank == MASTER_PROCESS: [ _save_linrate(ifgs, params, tiles, out_type=t) for t in ['linrate', 'linerror', 'linsamples'] ]
def main(params): """ Main workflow function for preparing interferograms for PyRate. :param dict params: Parameters dictionary read in from the config file """ # TODO: looks like ifg_paths are ordered according to ifg list # This probably won't be a problem because input list won't be reordered # and the original gamma generated list is ordered) this may not affect # the important pyrate stuff anyway, but might affect gen_thumbs.py. # Going to assume ifg_paths is ordered correcly # pylint: disable=too-many-branches shared.mpi_vs_multiprocess_logging("prepifg", params) ifg_paths = params[cf.INTERFEROGRAM_FILES] if params[cf.DEM_FILE] is not None: # optional DEM conversion ifg_paths.append(params[cf.DEM_FILE_PATH]) if params[cf.COH_MASK]: ifg_paths.extend(params[cf.COHERENCE_FILE_PATHS]) shared.mkdir_p(params[cf.OUT_DIR]) # create output dir user_exts = (params[cf.IFG_XFIRST], params[cf.IFG_YFIRST], params[cf.IFG_XLAST], params[cf.IFG_YLAST]) xlooks, ylooks, crop = cf.transform_params(params) ifgs = [prepifg_helper.dem_or_ifg(p.converted_path) for p in ifg_paths] exts = prepifg_helper.get_analysis_extent(crop, ifgs, xlooks, ylooks, user_exts=user_exts) process_ifgs_paths = np.array_split(ifg_paths, mpiops.size)[mpiops.rank] do_prepifg(process_ifgs_paths, exts, params) mpiops.comm.barrier() log.info("Finished prepifg")
def _prepifg_multiprocessing(m_path: MultiplePaths, exts: Tuple[float, float, float, float], params: dict): """ Multiprocessing wrapper for prepifg """ xlooks, ylooks, crop = cf.transform_params(params) thresh = params[cf.NO_DATA_AVERAGING_THRESHOLD] header = find_header(m_path, params) header[ifc.INPUT_TYPE] = m_path.input_type # If we're performing coherence masking, find the coherence file for this IFG. if params[cf.COH_MASK] and shared._is_interferogram(header): coherence_path = cf.coherence_paths_for(m_path.converted_path, params, tif=True) coherence_thresh = params[cf.COH_THRESH] else: coherence_path = None coherence_thresh = None if params[cf.LARGE_TIFS]: op = output_tiff_filename(m_path.converted_path, params[cf.OUT_DIR]) looks_path = cf.mlooked_path(op, ylooks, crop) return m_path.converted_path, coherence_path, looks_path else: prepifg_helper.prepare_ifg(m_path.converted_path, xlooks, ylooks, exts, thresh, crop, out_path=params[cf.OUT_DIR], header=header, coherence_path=coherence_path, coherence_thresh=coherence_thresh)
def test_vcm_legacy_vs_mpi(mpisync, tempdir, roipac_or_gamma_conf): params = configuration.Configuration(roipac_or_gamma_conf).__dict__ LEGACY_VCM_DIR = os.path.join(SML_TEST_DIR, 'vcm') legacy_vcm = np.genfromtxt(os.path.join(LEGACY_VCM_DIR, 'vcmt.csv'), delimiter=',') tmpdir = Path(mpiops.run_once(tempdir)) mpiops.run_once(common.copytree, params[cf.OBS_DIR], tmpdir) params[cf.OUT_DIR] = tmpdir.joinpath('out') params[cf.PARALLEL] = False xlks, ylks, crop = cf.transform_params(params) base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST], params[cf.OBS_DIR]) # dest_paths are tifs that have been geotif converted and multilooked dest_paths = cf.get_dest_paths(base_unw_paths, crop, params, xlks) # run conv2tif and prepifg, create the dest_paths files conv2tif.main(params) prepifg.main(params) tiles = pyrate.core.shared.get_tiles(dest_paths[0], rows=1, cols=1) preread_ifgs = process._create_ifg_dict(dest_paths, params=params, tiles=tiles) refpx, refpy = process._ref_pixel_calc(dest_paths, params) process._orb_fit_calc(dest_paths, params) process._ref_phase_estimation(dest_paths, params, refpx, refpy) maxvar, vcmt = process._maxvar_vcm_calc(dest_paths, params, preread_ifgs) np.testing.assert_array_almost_equal(maxvar, legacy_maxvar, decimal=4) np.testing.assert_array_almost_equal(legacy_vcm, vcmt, decimal=3) mpiops.run_once(shutil.rmtree, tmpdir)
def setUpClass(cls): params = Configuration(common.TEST_CONF_ROIPAC).__dict__ shutil.rmtree(params[cf.OUT_DIR]) params = Configuration(common.TEST_CONF_ROIPAC).__dict__ cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['prepifg.py', common.TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir params[cf.TMPDIR] = cls.temp_out_dir conv2tif.main(params) prepifg.main(params) params[cf.REF_EST_METHOD] = 1 params[cf.PARALLEL] = True xlks, ylks, crop = cf.transform_params(params) base_ifg_paths = [ c.unwrapped_path for c in params[cf.INTERFEROGRAM_FILES] ] headers = [roipac.roipac_header(i, params) for i in base_ifg_paths] dest_paths = [ Path(cls.temp_out_dir).joinpath(Path( c.sampled_path).name).as_posix() for c in params[cf.INTERFEROGRAM_FILES][:-2] ] # start run_pyrate copy ifgs = common.pre_prepare_ifgs(dest_paths, params) mst_grid = common.mst_calculation(dest_paths, params) # Estimate reference pixel location refx, refy = process._ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.core.orbital.remove_orbital_error(ifgs, params, headers) for i in ifgs: i.close() ifgs = common.pre_prepare_ifgs(dest_paths, params) for i in ifgs: i.close() cls.ref_phs, cls.ifgs = process._ref_phase_estimation( dest_paths, params, refx, refy)
def do_prepifg(gtiff_paths: List[str], params: dict) -> None: """ Prepare interferograms by applying multilooking/cropping operations. :param list gtiff_paths: List of full-res geotiffs :param dict params: Parameters dictionary corresponding to config file """ # pylint: disable=expression-not-assigned parallel = params[cf.PARALLEL] if mpiops.size > 1: parallel = False for f in gtiff_paths: if not os.path.isfile(f): raise FileNotFoundError("Can not find geotiff: " + str(f) + ". Ensure you have converted your " "interferograms to geotiffs.") ifgs = [prepifg_helper.dem_or_ifg(p) for p in gtiff_paths] xlooks, ylooks, crop = cf.transform_params(params) user_exts = (params[cf.IFG_XFIRST], params[cf.IFG_YFIRST], params[cf.IFG_XLAST], params[cf.IFG_YLAST]) exts = prepifg_helper.get_analysis_extent(crop, ifgs, xlooks, ylooks, user_exts=user_exts) thresh = params[cf.NO_DATA_AVERAGING_THRESHOLD] if params[cf.LARGE_TIFS]: log.info("Using gdal system calls to process prepifg") ifg = ifgs[0] res_str = [xlooks * ifg.x_step, ylooks * ifg.y_step] res_str = ' '.join([str(e) for e in res_str]) if parallel: Parallel(n_jobs=params[cf.PROCESSES], verbose=50)( delayed(__prepifg_system)( crop, exts, gtiff_path, params, res_str, thresh, xlooks, ylooks) for gtiff_path in gtiff_paths ) else: for gtiff_path in gtiff_paths: __prepifg_system(crop, exts, gtiff_path, params, res_str, thresh, xlooks, ylooks) else: if parallel: Parallel(n_jobs=params[cf.PROCESSES], verbose=50)( delayed(_prepifg_multiprocessing)(p, xlooks, ylooks, exts, thresh, crop, params) for p in gtiff_paths ) else: for gtiff_path in gtiff_paths: _prepifg_multiprocessing(gtiff_path, xlooks, ylooks, exts, thresh, crop, params)
def _merge_stack(rows, cols, params): """ Merge stacking outputs """ # setup paths xlks, _, crop = cf.transform_params(params) base_unw_paths = [] for p in Path(params[OUT_DIR]).rglob("*rlks_*cr.tif"): if "dem" not in str(p): base_unw_paths.append(str(p)) if not base_unw_paths: raise ConfigException( f"No rlooked files available in {params[OBS_DIR]} or {params[OBS_DIR]}" ) if "tif" in base_unw_paths[0].split(".")[1]: dest_tifs = base_unw_paths # cf.get_dest_paths(base_unw_paths, crop, params, xlks) for i, dest_tif in enumerate(dest_tifs): dest_tifs[i] = dest_tif.replace("_tif", "") else: dest_tifs = base_unw_paths # cf.get_dest_paths(base_unw_paths, crop, params, xlks) # load previously saved prepread_ifgs dict preread_ifgs_file = join(params[cf.TMPDIR], 'preread_ifgs.pk') ifgs = cp.load(open(preread_ifgs_file, 'rb')) tiles = shared.get_tiles(dest_tifs[0], rows, cols) # stacking aggregation if mpiops.size >= 3: [ _save_stack(ifgs, params, tiles, out_type=t) for i, t in enumerate( ['stack_rate', 'stack_error', 'stack_samples']) if i == mpiops.rank ] else: if mpiops.rank == MASTER_PROCESS: [ _save_stack(ifgs, params, tiles, out_type=t) for t in ['stack_rate', 'stack_error', 'stack_samples'] ] mpiops.comm.barrier()
def test_vcm_legacy_vs_mpi(mpisync, tempdir, get_config): from tests.common import SML_TEST_DIR, TEST_CONF_ROIPAC params_dict = get_config(TEST_CONF_ROIPAC) LEGACY_VCM_DIR = os.path.join(SML_TEST_DIR, 'vcm') legacy_vcm = np.genfromtxt(os.path.join(LEGACY_VCM_DIR, 'vcmt.csv'), delimiter=',') if mpiops.rank == 0: outdir = tempdir() else: outdir = None outdir = mpiops.comm.bcast(outdir, root=0) params_dict[cf.OUT_DIR] = outdir params_dict[cf.PARALLEL] = False xlks, ylks, crop = cf.transform_params(params_dict) base_unw_paths = cf.original_ifg_paths(params_dict[cf.IFG_FILE_LIST], params_dict[cf.OBS_DIR]) # dest_paths are tifs that have been geotif converted and multilooked dest_paths = cf.get_dest_paths(base_unw_paths, crop, params_dict, xlks) # run prepifg, create the dest_paths files if mpiops.rank == 0: conv2tif.main(params_dict) prepifg.main(params_dict) mpiops.comm.barrier() tiles = pyrate.core.shared.get_tiles(dest_paths[0], rows=1, cols=1) preread_ifgs = process._create_ifg_dict(dest_paths, params=params_dict, tiles=tiles) refpx, refpy = process._ref_pixel_calc(dest_paths, params_dict) process._orb_fit_calc(dest_paths, params_dict) process._ref_phase_estimation(dest_paths, params_dict, refpx, refpy) maxvar, vcmt = process._maxvar_vcm_calc(dest_paths, params_dict, preread_ifgs) np.testing.assert_array_almost_equal(maxvar, legacy_maxvar, decimal=4) np.testing.assert_array_almost_equal(legacy_vcm, vcmt, decimal=3) if mpiops.rank == 0: shutil.rmtree(outdir) common.remove_tifs(params_dict[cf.OBS_DIR])
def setUpClass(cls): params = cf.get_config_params(common.TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['prepifg.py', common.TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir params[cf.TMPDIR] = cls.temp_out_dir conv2tif.main(params) prepifg.main(params) params[cf.OUT_DIR] = cls.temp_out_dir params[cf.REF_EST_METHOD] = 2 params[cf.PARALLEL] = True xlks, ylks, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST], params[cf.OBS_DIR]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) # start run_pyrate copy ifgs = common.pre_prepare_ifgs(dest_paths, params) # Estimate reference pixel location refx, refy = process._ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.core.orbital.remove_orbital_error(ifgs, params) for i in ifgs: i.close() ifgs = common.pre_prepare_ifgs(dest_paths, params) for i in ifgs: i.close() cls.ref_phs, cls.ifgs = process._ref_phase_estimation( dest_paths, params, refx, refy)
def _prepifg_multiprocessing(m_path: MultiplePaths, exts: Tuple[float, float, float, float], params: dict): """ Multiprocessing wrapper for prepifg """ thresh = params[cf.NO_DATA_AVERAGING_THRESHOLD] hdr = find_header(m_path, params) hdr[ifc.INPUT_TYPE] = m_path.input_type xlooks, ylooks, crop = cf.transform_params(params) hdr[ifc.IFG_LKSX] = xlooks hdr[ifc.IFG_LKSY] = ylooks hdr[ifc.IFG_CROP] = crop # If we're performing coherence masking, find the coherence file for this IFG. if params[cf.COH_MASK] and shared._is_interferogram(hdr): coherence_path = cf.coherence_paths_for(m_path.converted_path, params, tif=True) coherence_thresh = params[cf.COH_THRESH] else: coherence_path = None coherence_thresh = None if params[cf.LARGE_TIFS]: return m_path.converted_path, coherence_path, m_path.sampled_path else: prepifg_helper.prepare_ifg(m_path.converted_path, xlooks, ylooks, exts, thresh, crop, out_path=m_path.sampled_path, header=hdr, coherence_path=coherence_path, coherence_thresh=coherence_thresh) Path(m_path.sampled_path).chmod(0o444) # readonly output
def do_prepifg(gtiff_paths, params): """ Prepare interferograms by applying multilooking/cropping operations. :param list gtiff_paths: List of full-res geotiffs :param dict params: Parameters dictionary corresponding to config file """ # pylint: disable=expression-not-assigned log.info("Preparing interferograms by cropping/multilooking") parallel = params[cf.PARALLEL] for f in gtiff_paths: if not os.path.isfile(f): raise Exception( "Can not find geotiff: " + str(f) + ". Ensure you have converted your interferograms to geotiffs.") ifgs = [prepifg_helper.dem_or_ifg(p) for p in gtiff_paths] xlooks, ylooks, crop = cf.transform_params(params) user_exts = (params[cf.IFG_XFIRST], params[cf.IFG_YFIRST], params[cf.IFG_XLAST], params[cf.IFG_YLAST]) exts = prepifg_helper.get_analysis_extent(crop, ifgs, xlooks, ylooks, user_exts=user_exts) log.debug("Extents (xmin, ymin, xmax, ymax): " + str(exts)) thresh = params[cf.NO_DATA_AVERAGING_THRESHOLD] if parallel: Parallel(n_jobs=params[cf.PROCESSES], verbose=50)(delayed(_prepifg_multiprocessing)( p, xlooks, ylooks, exts, thresh, crop, params) for p in gtiff_paths) else: [ _prepifg_multiprocessing(p, xlooks, ylooks, exts, thresh, crop, params) for p in gtiff_paths ]
def setUpClass(cls): cls.tif_dir = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_GAMMA # change the required params cls.params = Configuration(cls.test_conf).__dict__ cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA cls.params[cf.PROCESSOR] = 1 # gamma file_list = list( cf.parse_namelist(os.path.join(common.SML_TEST_GAMMA, 'ifms_17'))) fd, cls.params[cf.IFG_FILE_LIST] = tempfile.mkstemp(suffix='.conf', dir=cls.tif_dir) os.close(fd) # write a short filelist with only 3 gamma unws with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp: for f in file_list[:3]: fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n') cls.params[cf.OUT_DIR] = cls.tif_dir cls.params[cf.PARALLEL] = 0 cls.params[cf.REF_EST_METHOD] = 1 cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA # base_unw_paths need to be geotiffed and multilooked by run_prepifg cls.base_unw_paths = cf.original_ifg_paths( cls.params[cf.IFG_FILE_LIST], cls.params[cf.OBS_DIR]) cls.base_unw_paths.append(common.SML_TEST_DEM_GAMMA) xlks, ylks, crop = cf.transform_params(cls.params) # dest_paths are tifs that have been geotif converted and multilooked conv2tif.main(cls.params) prepifg.main(cls.params) # run_prepifg.gamma_prepifg(cls.base_unw_paths, cls.params) cls.base_unw_paths.pop() # removed dem as we don't want it in ifgs cls.dest_paths = cf.get_dest_paths(cls.base_unw_paths, crop, cls.params, xlks) cls.ifgs = common.small_data_setup(datafiles=cls.dest_paths)
def setUpClass(cls): params = Configuration(TEST_CONF_ROIPAC).__dict__ cls.temp_out_dir = tempfile.mkdtemp() params[cf.OUT_DIR] = cls.temp_out_dir params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) shared.mkdir_p(params[cf.TMPDIR]) conv2tif.main(params) prepifg.main(params) params[cf.REF_EST_METHOD] = 2 xlks, _, crop = cf.transform_params(params) base_ifg_paths = [ c.unwrapped_path for c in params[cf.INTERFEROGRAM_FILES] ] headers = [roipac.roipac_header(i, params) for i in base_ifg_paths] dest_paths = [ Path(cls.temp_out_dir).joinpath(Path( c.sampled_path).name).as_posix() for c in params[cf.INTERFEROGRAM_FILES][:-2] ] # start run_pyrate copy ifgs = pre_prepare_ifgs(dest_paths, params) mst_grid = tests.common.mst_calculation(dest_paths, params) refx, refy = process._ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.core.orbital.remove_orbital_error(ifgs, params, headers) ifgs = prepare_ifgs_without_phase(dest_paths, params) for ifg in ifgs: ifg.close() _, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy) ifgs[0].open() r_dist = vcm_module.RDist(ifgs[0])() ifgs[0].close() maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in dest_paths] for ifg in ifgs: ifg.open() vcmt = vcm_module.get_vcmt(ifgs, maxvar) for ifg in ifgs: ifg.close() ifg.open() # Calculate stacked rate map params[cf.PARALLEL] = 1 cls.rate, cls.error, cls.samples = tests.common.calculate_stack_rate( ifgs, params, vcmt, mst_mat=mst_grid) # Calculate stacked rate map params[cf.PARALLEL] = 0 cls.rate_s, cls.error_s, cls.samples_s = tests.common.calculate_stack_rate( ifgs, params, vcmt, mst_mat=mst_grid) stackrate_dir = os.path.join(SML_TEST_DIR, 'stackrate') cls.rate_container = np.genfromtxt(os.path.join( stackrate_dir, 'stackmap.csv'), delimiter=',') cls.error_container = np.genfromtxt(os.path.join( stackrate_dir, 'errormap.csv'), delimiter=',') cls.samples_container = np.genfromtxt(os.path.join( stackrate_dir, 'coh_sta.csv'), delimiter=',') for ifg in ifgs: ifg.close()
def _merge_timeseries(rows, cols, params): """ Merge time series output """ xlks, _, crop = cf.transform_params(params) base_unw_paths = [] for p in Path(params[OUT_DIR]).rglob("*rlks_*cr.tif"): if "dem" not in str(p): base_unw_paths.append(str(p)) if "tif" in base_unw_paths[0].split(".")[1]: dest_tifs = base_unw_paths # cf.get_dest_paths(base_unw_paths, crop, params, xlks) for i, dest_tif in enumerate(dest_tifs): dest_tifs[i] = dest_tif.replace("_tif", "") else: dest_tifs = base_unw_paths # cf.get_dest_paths(base_unw_paths, crop, params, xlks) output_dir = params[cf.TMPDIR] # load previously saved prepread_ifgs dict preread_ifgs_file = join(output_dir, 'preread_ifgs.pk') ifgs = cp.load(open(preread_ifgs_file, 'rb')) # metadata and projections gt, md, wkt = ifgs['gt'], ifgs['md'], ifgs['wkt'] epochlist = ifgs['epochlist'] ifgs = [v for v in ifgs.values() if isinstance(v, PrereadIfg)] tiles = shared.get_tiles(dest_tifs[0], rows, cols) # load the first tsincr file to determine the number of time series tifs tsincr_file = os.path.join(output_dir, 'tsincr_0.npy') tsincr = np.load(file=tsincr_file) # pylint: disable=no-member no_ts_tifs = tsincr.shape[2] # we create 2 x no_ts_tifs as we are splitting tsincr and tscuml # to all processes. process_tifs = mpiops.array_split(range(2 * no_ts_tifs)) # depending on nvelpar, this will not fit in memory # e.g. nvelpar=100, nrows=10000, ncols=10000, 32bit floats need 40GB memory # 32 * 100 * 10000 * 10000 / 8 bytes = 4e10 bytes = 40 GB # the double for loop helps us overcome the memory limit log.info('Process {} writing {} timeseries tifs of ' 'total {}'.format(mpiops.rank, len(process_tifs), no_ts_tifs * 2)) for i in process_tifs: tscum_g = np.empty(shape=ifgs[0].shape, dtype=np.float32) if i < no_ts_tifs: for n, t in enumerate(tiles): _assemble_tiles(i, n, t, tscum_g, output_dir, 'tscuml') md[ifc.EPOCH_DATE] = epochlist.dates[i + 1] # sequence position; first time slice is #0 md['SEQUENCE_POSITION'] = i + 1 dest = os.path.join( params[cf.OUT_DIR], 'tscuml' + "_" + str(epochlist.dates[i + 1]) + ".tif") md[ifc.DATA_TYPE] = ifc.CUML shared.write_output_geotiff(md, gt, wkt, tscum_g, dest, np.nan) else: tsincr_g = np.empty(shape=ifgs[0].shape, dtype=np.float32) i %= no_ts_tifs for n, t in enumerate(tiles): _assemble_tiles(i, n, t, tsincr_g, output_dir, 'tsincr') md[ifc.EPOCH_DATE] = epochlist.dates[i + 1] # sequence position; first time slice is #0 md['SEQUENCE_POSITION'] = i + 1 dest = os.path.join( params[cf.OUT_DIR], 'tsincr' + "_" + str(epochlist.dates[i + 1]) + ".tif") md[ifc.DATA_TYPE] = ifc.INCR shared.write_output_geotiff(md, gt, wkt, tsincr_g, dest, np.nan) mpiops.comm.barrier() log.debug('Process {} finished writing {} timeseries tifs of ' 'total {}'.format(mpiops.rank, len(process_tifs), no_ts_tifs * 2))
def setUpClass(cls): params = cf.get_config_params(TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() params[cf.OUT_DIR] = cls.temp_out_dir params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) shared.mkdir_p(params[cf.TMPDIR]) conv2tif.main(params) prepifg.main(params) params[cf.REF_EST_METHOD] = 2 xlks, _, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST], params[cf.OBS_DIR]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) print(f"base_ifg_paths={base_ifg_paths}") print(f"dest_paths={dest_paths}") # start run_pyrate copy ifgs = pre_prepare_ifgs(dest_paths, params) mst_grid = tests.common.mst_calculation(dest_paths, params) refx, refy = process._ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.core.orbital.remove_orbital_error(ifgs, params) ifgs = prepare_ifgs_without_phase(dest_paths, params) for ifg in ifgs: ifg.close() _, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy) ifgs[0].open() r_dist = vcm_module.RDist(ifgs[0])() ifgs[0].close() maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in dest_paths] for ifg in ifgs: ifg.open() vcmt = vcm_module.get_vcmt(ifgs, maxvar) for ifg in ifgs: ifg.close() ifg.open() # Calculate linear rate map params[cf.PARALLEL] = 1 cls.rate, cls.error, cls.samples = tests.common.calculate_linear_rate( ifgs, params, vcmt, mst_mat=mst_grid) params[cf.PARALLEL] = 2 cls.rate_2, cls.error_2, cls.samples_2 = \ tests.common.calculate_linear_rate(ifgs, params, vcmt, mst_mat=mst_grid) params[cf.PARALLEL] = 0 # Calculate linear rate map cls.rate_s, cls.error_s, cls.samples_s = \ tests.common.calculate_linear_rate(ifgs, params, vcmt, mst_mat=mst_grid) linrate_dir = os.path.join(SML_TEST_DIR, 'linrate') cls.rate_container = np.genfromtxt(os.path.join( linrate_dir, 'stackmap.csv'), delimiter=',') cls.error_container = np.genfromtxt(os.path.join( linrate_dir, 'errormap.csv'), delimiter=',') cls.samples_container = np.genfromtxt(os.path.join( linrate_dir, 'coh_sta.csv'), delimiter=',') for ifg in ifgs: ifg.close()
def setup_class(cls): params = Configuration(common.TEST_CONF_ROIPAC).__dict__ params[cf.TEMP_MLOOKED_DIR] = os.path.join(params[cf.OUT_DIR], cf.TEMP_MLOOKED_DIR) conv2tif.main(params) prepifg.main(params) params[cf.REF_EST_METHOD] = 2 xlks, _, crop = cf.transform_params(params) dest_paths, headers = common.repair_params_for_correct_tests( params[cf.OUT_DIR], params) correct._copy_mlooked(params) copied_dest_paths = [ os.path.join(params[cf.TEMP_MLOOKED_DIR], os.path.basename(d)) for d in dest_paths ] del dest_paths # start run_pyrate copy ifgs = common.pre_prepare_ifgs(copied_dest_paths, params) mst_grid = common.mst_calculation(copied_dest_paths, params) refx, refy = pyrate.core.refpixel.ref_pixel_calc_wrapper(params) params[cf.REFX] = refx params[cf.REFY] = refy params[cf.ORBFIT_OFFSET] = True # Estimate and remove orbit errors pyrate.core.orbital.remove_orbital_error(ifgs, params) ifgs = common.prepare_ifgs_without_phase(copied_dest_paths, params) for ifg in ifgs: ifg.close() correct._update_params_with_tiles(params) _, ifgs = pyrate.core.ref_phs_est.ref_phase_est_wrapper(params) ifgs[0].open() r_dist = covariance.RDist(ifgs[0])() ifgs[0].close() # Calculate interferogram noise maxvar = [ covariance.cvd(i, params, r_dist)[0] for i in copied_dest_paths ] for ifg in ifgs: ifg.open() vcmt = covariance.get_vcmt(ifgs, maxvar) for ifg in ifgs: ifg.close() ifg.open() ifg.nodata_value = 0.0 params[cf.TIME_SERIES_METHOD] = 2 params[cf.PARALLEL] = 1 # Calculate time series cls.tsincr, cls.tscum, _ = common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid) params[cf.PARALLEL] = 0 # Calculate time series serailly by the pixel cls.tsincr_0, cls.tscum_0, _ = common.calculate_time_series( ifgs, params, vcmt, mst=mst_grid) # copy legacy data SML_TIME_SERIES_DIR = os.path.join(common.SML_TEST_DIR, 'time_series') tsincr_path = os.path.join(SML_TIME_SERIES_DIR, 'ts_incr_interp0_method2.csv') ts_incr = np.genfromtxt(tsincr_path) tscum_path = os.path.join(SML_TIME_SERIES_DIR, 'ts_cum_interp0_method2.csv') ts_cum = np.genfromtxt(tscum_path) cls.ts_incr = np.reshape(ts_incr, newshape=cls.tsincr_0.shape, order='F') cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F') cls.params = params
def setUpClass(cls): rate_types = ['linrate', 'linerror', 'linsamples'] cls.tif_dir = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_GAMMA # change the required params params = cf.get_config_params(cls.test_conf) params[cf.OBS_DIR] = common.SML_TEST_GAMMA params[cf.PROCESSOR] = 1 # gamma params[cf.IFG_FILE_LIST] = os.path.join( common.SML_TEST_GAMMA, 'ifms_17') params[cf.OUT_DIR] = cls.tif_dir params[cf.PARALLEL] = 1 params[cf.APS_CORRECTION] = False params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) xlks, ylks, crop = cf.transform_params(params) # base_unw_paths need to be geotiffed by converttogeotif # and multilooked by run_prepifg base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST], params[cf.OBS_DIR]) # dest_paths are tifs that have been geotif converted and multilooked cls.dest_paths = cf.get_dest_paths( base_unw_paths, crop, params, xlks) gtif_paths = conv2tif.do_geotiff(base_unw_paths, params) prepifg.do_prepifg(gtif_paths, params) tiles = pyrate.core.shared.get_tiles(cls.dest_paths[0], 3, 3) ifgs = common.small_data_setup() cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = \ process.process_ifgs(cls.dest_paths, params, 3, 3) cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) cls.rate_p, cls.error_p, cls.samples_p = [ common.reconstruct_linrate( ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types ] common.remove_tifs(params[cf.OBS_DIR]) # now create the non parallel version cls.tif_dir_s = tempfile.mkdtemp() params[cf.PARALLEL] = 0 params[cf.OUT_DIR] = cls.tif_dir_s params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) cls.dest_paths_s = cf.get_dest_paths( base_unw_paths, crop, params, xlks) gtif_paths = conv2tif.do_geotiff(base_unw_paths, params) prepifg.do_prepifg(gtif_paths, params) cls.refpixel, cls.maxvar, cls.vcmt = \ process.process_ifgs(cls.dest_paths_s, params, 3, 3) cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) cls.rate, cls.error, cls.samples = [ common.reconstruct_linrate( ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types ]
def test_transform_params(): params = {config.IFG_LKSX: 3, config.IFG_LKSY: 2, config.IFG_CROP_OPT: 1} assert cf.transform_params(params) == (3, 2, 1)