def postprocess_linrate(rows, cols, params): """ Postprocess linear rate. :param rows: xxxx :param cols: xxxx :param params: xxxx :return xxxx """ # pylint: disable=expression-not-assigned # setup paths xlks, _, crop = cf.transform_params(params) base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) dest_tifs = cf.get_dest_paths(base_unw_paths, crop, params, xlks) # load previously saved prepread_ifgs dict preread_ifgs_file = join(params[cf.TMPDIR], 'preread_ifgs.pk') ifgs = cp.load(open(preread_ifgs_file, 'rb')) tiles = run_pyrate.get_tiles(dest_tifs[0], rows, cols) # linrate aggregation if mpiops.size >= 3: [ save_linrate(ifgs, params, tiles, out_type=t) for i, t in enumerate(['linrate', 'linerror', 'linsamples']) if i == mpiops.rank ] else: if mpiops.rank == MASTER_PROCESS: [ save_linrate(ifgs, params, tiles, out_type=t) for t in ['linrate', 'linerror', 'linsamples'] ]
def setUpClass(cls): cls.tif_dir = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_GAMMA # change the required params cls.params = cf.get_config_params(cls.test_conf) cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA cls.params[cf.PROCESSOR] = 1 # gamma file_list = list(cf.parse_namelist(os.path.join(common.SML_TEST_GAMMA, 'ifms_17'))) fd, cls.params[cf.IFG_FILE_LIST] = tempfile.mkstemp(suffix='.conf', dir=cls.tif_dir) os.close(fd) # write a short filelist with only 3 gamma unws with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp: for f in file_list[:3]: fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n') cls.params[cf.OUT_DIR] = cls.tif_dir cls.params[cf.PARALLEL] = 0 cls.params[cf.REF_EST_METHOD] = 1 cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA # base_unw_paths need to be geotiffed and multilooked by run_prepifg cls.base_unw_paths = cf.original_ifg_paths( cls.params[cf.IFG_FILE_LIST]) cls.base_unw_paths.append(common.SML_TEST_DEM_GAMMA) xlks, ylks, crop = cf.transform_params(cls.params) # dest_paths are tifs that have been geotif converted and multilooked run_prepifg.gamma_prepifg(cls.base_unw_paths, cls.params) cls.base_unw_paths.pop() # removed dem as we don't want it in ifgs dest_paths = cf.get_dest_paths( cls.base_unw_paths, crop, cls.params, xlks) cls.ifgs = common.small_data_setup(datafiles=dest_paths)
def setUpClass(cls): params = cf.get_config_params(common.TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['run_prepifg.py', common.TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir run_prepifg.main(params) params[cf.REF_EST_METHOD] = 2 xlks, ylks, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) # start run_pyrate copy ifgs = common.pre_prepare_ifgs(dest_paths, params) mst_grid = common.mst_calculation(dest_paths, params) refx, refy = run_pyrate._ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.orbital.remove_orbital_error(ifgs, params) ifgs = common.prepare_ifgs_without_phase(dest_paths, params) _, ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy) r_dist = covariance.RDist(ifgs[0])() maxvar = [covariance.cvd(i, params, r_dist)[0] for i in ifgs] vcmt = covariance.get_vcmt(ifgs, maxvar) params[cf.TIME_SERIES_METHOD] = 1 params[cf.PARALLEL] = 0 # Calculate time series cls.tsincr_0, cls.tscum_0, _ = common.calculate_time_series( ifgs, params, vcmt, mst=mst_grid) params[cf.PARALLEL] = 1 cls.tsincr_1, cls.tscum_1, cls.tsvel_1 = \ common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid) params[cf.PARALLEL] = 2 cls.tsincr_2, cls.tscum_2, cls.tsvel_2 = \ common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid) # load the matlab data ts_dir = os.path.join(common.SML_TEST_DIR, 'matlab_time_series') tsincr_path = os.path.join(ts_dir, 'ts_incr_interp0_method1.csv') ts_incr = np.genfromtxt(tsincr_path) # the matlab tsvel return is a bit pointless and not tested here # tserror is not returned # tserr_path = os.path.join(SML_TIME_SERIES_DIR, # 'ts_error_interp0_method1.csv') # ts_err = np.genfromtxt(tserr_path, delimiter=',') tscum_path = os.path.join(ts_dir, 'ts_cum_interp0_method1.csv') ts_cum = np.genfromtxt(tscum_path) cls.ts_incr = np.reshape(ts_incr, newshape=cls.tsincr_0.shape, order='F') cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F')
def main(params=None): """ xxxx :param params: Parameters dictionary read in from the config file :return xxxx """ # TODO: looks like base_ifg_paths are ordered according to ifg list # This probably won't be a problem because input list won't be reordered # and the original gamma generated list is ordered) this may not affect # the important pyrate stuff anyway, but might affect gen_thumbs.py. # Going to assume base_ifg_paths is ordered correcly usage = 'Usage: pyrate prepifg <config_file>' if mpiops.size > 1: # Over-ride input options if this is an MPI job params[cf.LUIGI] = False params[cf.PARALLEL] = False if params: base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) use_luigi = params[cf.LUIGI] # luigi or no luigi if use_luigi: raise cf.ConfigException('params can not be provided with luigi') else: # if params not provided read from config file if (not params) and (len(sys.argv) < 3): print(usage) return base_ifg_paths, _, params = cf.get_ifg_paths(sys.argv[2]) use_luigi = params[cf.LUIGI] # luigi or no luigi raw_config_file = sys.argv[2] base_ifg_paths.append(params[cf.DEM_FILE]) processor = params[cf.PROCESSOR] # roipac or gamma if processor == GAMMA: # Incidence/elevation only supported for GAMMA if params[cf.APS_INCIDENCE_MAP]: base_ifg_paths.append(params[cf.APS_INCIDENCE_MAP]) if params[cf.APS_ELEVATION_MAP]: base_ifg_paths.append(params[cf.APS_ELEVATION_MAP]) if use_luigi: log.info("Running prepifg using luigi") luigi.configuration.LuigiConfigParser.add_config_path( pythonify_config(raw_config_file)) luigi.build([PrepareInterferograms()], local_scheduler=True) else: process_base_ifgs_paths = \ np.array_split(base_ifg_paths, mpiops.size)[mpiops.rank] if processor == ROIPAC: roipac_prepifg(process_base_ifgs_paths, params) elif processor == GAMMA: gamma_prepifg(process_base_ifgs_paths, params) else: raise prepifg.PreprocessError('Processor must be ROI_PAC (0) or ' 'GAMMA (1)') log.info("Finished prepifg")
def setUpClass(cls): rate_types = ['linrate', 'linerror', 'linsamples'] cls.tif_dir = tempfile.mkdtemp() cls.test_conf = common.TEST_CONF_GAMMA # change the required params params = cf.get_config_params(cls.test_conf) params[cf.OBS_DIR] = common.SML_TEST_GAMMA params[cf.PROCESSOR] = 1 # gamma params[cf.IFG_FILE_LIST] = os.path.join(common.SML_TEST_GAMMA, 'ifms_17') params[cf.OUT_DIR] = cls.tif_dir params[cf.PARALLEL] = 0 params[cf.APS_CORRECTION] = False params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) xlks, ylks, crop = cf.transform_params(params) # base_unw_paths need to be geotiffed and multilooked by run_prepifg base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) # dest_paths are tifs that have been geotif converted and multilooked cls.dest_paths = cf.get_dest_paths(base_unw_paths, crop, params, xlks) run_prepifg.gamma_prepifg(base_unw_paths, params) tiles = run_pyrate.get_tiles(cls.dest_paths[0], 3, 3) ifgs = common.small_data_setup() cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = \ run_pyrate.process_ifgs(cls.dest_paths, params, 3, 3) cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) cls.rate_p, cls.error_p, cls.samples_p = [ common.reconstruct_linrate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types ] # now create the non parallel version cls.tif_dir_s = tempfile.mkdtemp() params[cf.PARALLEL] = 0 params[cf.OUT_DIR] = cls.tif_dir_s params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) cls.dest_paths_s = cf.get_dest_paths(base_unw_paths, crop, params, xlks) run_prepifg.gamma_prepifg(base_unw_paths, params) cls.refpixel, cls.maxvar, cls.vcmt = \ run_pyrate.process_ifgs(cls.dest_paths_s, params, 3, 3) cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) cls.rate, cls.error, cls.samples = [ common.reconstruct_linrate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types ]
def main(params=None): """ Main workflow function for preparing interferograms for PyRate. :param dict params: Parameters dictionary read in from the config file """ # TODO: looks like base_ifg_paths are ordered according to ifg list # This probably won't be a problem because input list won't be reordered # and the original gamma generated list is ordered) this may not affect # the important pyrate stuff anyway, but might affect gen_thumbs.py. # Going to assume base_ifg_paths is ordered correcly # pylint: disable=too-many-branches usage = 'Usage: pyrate prepifg <config_file>' if mpiops.size > 1: # Over-ride input options if this is an MPI job params[cf.PARALLEL] = False if params: base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) else: # if params not provided read from config file if (not params) and (len(sys.argv) < 3): print(usage) return base_ifg_paths, _, params = cf.get_ifg_paths(sys.argv[2]) base_ifg_paths.append(params[cf.DEM_FILE]) processor = params[cf.PROCESSOR] # roipac or gamma if processor == GAMMA: # Incidence/elevation only supported for GAMMA if params[cf.APS_INCIDENCE_MAP]: base_ifg_paths.append(params[cf.APS_INCIDENCE_MAP]) if params[cf.APS_ELEVATION_MAP]: base_ifg_paths.append(params[cf.APS_ELEVATION_MAP]) mkdir_p(params[cf.OUT_DIR]) # create output dir process_base_ifgs_paths = np.array_split(base_ifg_paths, mpiops.size)[mpiops.rank] if processor == ROIPAC: roipac_prepifg(process_base_ifgs_paths, params) elif processor == GAMMA: gamma_prepifg(process_base_ifgs_paths, params) else: raise prepifg.PreprocessError( 'Processor must be ROI_PAC (0) or GAMMA (1)') log.info("Finished prepifg")
def setUpClass(cls): params = cf.get_config_params(TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['run_prepifg.py', TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir params[cf.REF_EST_METHOD] = 2 run_prepifg.main(params) xlks, ylks, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) ifgs = shared.pre_prepare_ifgs(dest_paths, params) refx, refy = run_pyrate.ref_pixel_calc(dest_paths, params) pyrate.orbital.remove_orbital_error(ifgs, params) ifgs = prepare_ifgs_without_phase(dest_paths, params) _, ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy) # Calculate interferogram noise cls.maxvar = [cvd(i, params)[0] for i in ifgs] cls.vcmt = get_vcmt(ifgs, cls.maxvar)
def setUpClass(cls): params = cf.get_config_params(common.TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['run_prepifg.py', common.TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir run_prepifg.main(params) params[cf.OUT_DIR] = cls.temp_out_dir params[cf.REF_EST_METHOD] = 2 params[cf.PARALLEL] = False xlks, ylks, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) # start run_pyrate copy ifgs = shared.pre_prepare_ifgs(dest_paths, params) mst_grid = tests.common.mst_calculation(dest_paths, params) # Estimate reference pixel location refx, refy = run_pyrate.ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.orbital.remove_orbital_error(ifgs, params) for i in ifgs: i.close() ifgs = shared.pre_prepare_ifgs(dest_paths, params) cls.ref_phs, cls.ifgs = estimate_ref_phase(ifgs, params, refx, refy) # end run_pyrate copy for i in ifgs: i.close()
def test_vcm_matlab_vs_mpi(mpisync, tempdir, get_config): from tests.common import SML_TEST_DIR, TEST_CONF_ROIPAC params_dict = get_config(TEST_CONF_ROIPAC) MATLAB_VCM_DIR = os.path.join(SML_TEST_DIR, 'matlab_vcm') matlab_vcm = np.genfromtxt(os.path.join(MATLAB_VCM_DIR, 'matlab_vcmt.csv'), delimiter=',') if mpiops.rank == 0: outdir = tempdir() else: outdir = None outdir = mpiops.comm.bcast(outdir, root=0) params_dict[cf.OUT_DIR] = outdir params_dict[cf.PARALLEL] = False xlks, ylks, crop = cf.transform_params(params_dict) base_unw_paths = cf.original_ifg_paths(params_dict[cf.IFG_FILE_LIST]) # dest_paths are tifs that have been geotif converted and multilooked dest_paths = cf.get_dest_paths(base_unw_paths, crop, params_dict, xlks) # run prepifg, create the dest_paths files if mpiops.rank == 0: run_prepifg.roipac_prepifg(base_unw_paths, params_dict) mpiops.comm.barrier() tiles = run_pyrate.get_tiles(dest_paths[0], rows=1, cols=1) preread_ifgs = run_pyrate.create_ifg_dict(dest_paths, params=params_dict, tiles=tiles) refpx, refpy = run_pyrate.ref_pixel_calc(dest_paths, params_dict) run_pyrate.orb_fit_calc(dest_paths, params_dict) run_pyrate.ref_phase_estimation(dest_paths, params_dict, refpx, refpy) maxvar, vcmt = run_pyrate.maxvar_vcm_calc(dest_paths, params_dict, preread_ifgs) np.testing.assert_array_almost_equal(maxvar, matlab_maxvar, decimal=4) np.testing.assert_array_almost_equal(matlab_vcm, vcmt, decimal=3) if mpiops.rank == 0: shutil.rmtree(outdir)
def setUpClass(cls): params = cf.get_config_params(TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['run_prepifg.py', TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir params[cf.TMPDIR] = os.path.join(cls.temp_out_dir, cf.TMPDIR) shared.mkdir_p(params[cf.TMPDIR]) params[cf.REF_EST_METHOD] = 2 run_prepifg.main(params) cls.params = params xlks, ylks, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) ifgs = common.pre_prepare_ifgs(dest_paths, params) refx, refy = run_pyrate._ref_pixel_calc(dest_paths, params) pyrate.orbital.remove_orbital_error(ifgs, params) ifgs = prepare_ifgs_without_phase(dest_paths, params) _, cls.ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy) r_dist = RDist(ifgs[0])() # Calculate interferogram noise cls.maxvar = [cvd(i, params, r_dist, calc_alpha=True, save_acg=True, write_vals=True)[0] for i in ifgs] cls.vcmt = get_vcmt(ifgs, cls.maxvar)
def setUpClass(cls): params = cf.get_config_params(TEST_CONF_ROIPAC) cls.temp_out_dir = tempfile.mkdtemp() sys.argv = ['run_prepifg.py', TEST_CONF_ROIPAC] params[cf.OUT_DIR] = cls.temp_out_dir params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) shared.mkdir_p(params[cf.TMPDIR]) run_prepifg.main(params) params[cf.REF_EST_METHOD] = 2 xlks, _, crop = cf.transform_params(params) base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks) # start run_pyrate copy ifgs = pre_prepare_ifgs(dest_paths, params) mst_grid = tests.common.mst_calculation(dest_paths, params) refx, refy = run_pyrate._ref_pixel_calc(dest_paths, params) # Estimate and remove orbit errors pyrate.orbital.remove_orbital_error(ifgs, params) ifgs = prepare_ifgs_without_phase(dest_paths, params) _, ifgs = rpe.estimate_ref_phase(ifgs, params, refx, refy) r_dist = vcm_module.RDist(ifgs[0])() maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in ifgs] vcmt = vcm_module.get_vcmt(ifgs, maxvar) # Calculate linear rate map params[cf.PARALLEL] = 1 cls.rate, cls.error, cls.samples = tests.common.calculate_linear_rate( ifgs, params, vcmt, mst_mat=mst_grid) params[cf.PARALLEL] = 2 cls.rate_2, cls.error_2, cls.samples_2 = \ tests.common.calculate_linear_rate(ifgs, params, vcmt, mst_mat=mst_grid) params[cf.PARALLEL] = 0 # Calculate linear rate map cls.rate_s, cls.error_s, cls.samples_s = \ tests.common.calculate_linear_rate(ifgs, params, vcmt, mst_mat=mst_grid) matlab_linrate_dir = os.path.join(SML_TEST_DIR, 'matlab_linrate') cls.rate_matlab = np.genfromtxt(os.path.join(matlab_linrate_dir, 'stackmap.csv'), delimiter=',') cls.error_matlab = np.genfromtxt(os.path.join(matlab_linrate_dir, 'errormap.csv'), delimiter=',') cls.samples_matlab = np.genfromtxt(os.path.join( matlab_linrate_dir, 'coh_sta.csv'), delimiter=',')
def test_timeseries_linrate_mpi(mpisync, tempdir, modify_config, ref_est_method, row_splits, col_splits, get_crop, orbfit_lks, orbfit_method, orbfit_degrees): params = modify_config outdir = mpiops.run_once(tempdir) params[cf.OUT_DIR] = outdir params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) params[cf.DEM_HEADER_FILE] = SML_TEST_DEM_HDR_GAMMA params[cf.REF_EST_METHOD] = ref_est_method params[cf.IFG_CROP_OPT] = get_crop params[cf.ORBITAL_FIT_LOOKS_Y] = orbfit_lks params[cf.ORBITAL_FIT_LOOKS_X] = orbfit_lks params[cf.ORBITAL_FIT_METHOD] = orbfit_method params[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees xlks, ylks, crop = cf.transform_params(params) if xlks * col_splits > 45 or ylks * row_splits > 70: print('skipping test because lks and col_splits are not compatible') return # skip some tests in travis to run CI faster if TRAVIS and (xlks % 2 or row_splits % 2 or col_splits % 2 or orbfit_lks % 2): print('Skipping in travis env for faster CI run') return print("xlks={}, ref_est_method={}, row_splits={}, col_splits={}, " "get_crop={}, orbfit_lks={}, orbfit_method={}, " "rank={}".format(xlks, ref_est_method, row_splits, col_splits, get_crop, orbfit_lks, orbfit_method, orbfit_degrees, mpiops.rank)) base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) # dest_paths are tifs that have been geotif converted and multilooked dest_paths = cf.get_dest_paths(base_unw_paths, crop, params, xlks) # run prepifg, create the dest_paths files if mpiops.rank == 0: run_prepifg.gamma_prepifg(base_unw_paths, params) mpiops.comm.barrier() (refpx, refpy), maxvar, vcmt = run_pyrate.process_ifgs(ifg_paths=dest_paths, params=params, rows=row_splits, cols=col_splits) tiles = mpiops.run_once(run_pyrate.get_tiles, dest_paths[0], rows=row_splits, cols=col_splits) postprocessing.postprocess_linrate(row_splits, col_splits, params) postprocessing.postprocess_timeseries(row_splits, col_splits, params) ifgs_mpi_out_dir = params[cf.OUT_DIR] ifgs_mpi = small_data_setup(datafiles=dest_paths) # single process timeseries/linrate calculation if mpiops.rank == 0: params_old = modify_config params_old[cf.OUT_DIR] = tempdir() params_old[cf.REF_EST_METHOD] = ref_est_method params_old[cf.IFG_CROP_OPT] = get_crop params_old[cf.ORBITAL_FIT_LOOKS_Y] = orbfit_lks params_old[cf.ORBITAL_FIT_LOOKS_X] = orbfit_lks params_old[cf.ORBITAL_FIT_METHOD] = orbfit_method params_old[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees xlks, ylks, crop = cf.transform_params(params_old) base_unw_paths = cf.original_ifg_paths(params_old[cf.IFG_FILE_LIST]) dest_paths = cf.get_dest_paths(base_unw_paths, crop, params_old, xlks) run_prepifg.gamma_prepifg(base_unw_paths, params_old) ifgs = shared.pre_prepare_ifgs(dest_paths, params_old) mst_grid = tests.common.mst_calculation(dest_paths, params_old) refy, refx = refpixel.ref_pixel(ifgs, params_old) assert (refx == refpx) and (refy == refpy) # both must match pyrate.orbital.remove_orbital_error(ifgs, params_old) ifgs = common.prepare_ifgs_without_phase(dest_paths, params_old) rpe.estimate_ref_phase(ifgs, params_old, refx, refy) ifgs = shared.pre_prepare_ifgs(dest_paths, params_old) maxvar_s = [vcm.cvd(i, params_old)[0] for i in ifgs] vcmt_s = vcm.get_vcmt(ifgs, maxvar) tsincr, tscum, _ = tests.common.compute_time_series( ifgs, mst_grid, params, vcmt) rate, error, samples = tests.common.calculate_linear_rate( ifgs, params_old, vcmt, mst_grid) mst_mpi = reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) np.testing.assert_array_almost_equal(mst_grid, mst_mpi) tsincr_mpi, tscum_mpi = reconstruct_times_series( ifgs[0].shape, tiles, params[cf.TMPDIR]) rate_mpi, error_mpi, samples_mpi = \ [reconstruct_linrate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in ['linrate', 'linerror', 'linsamples']] np.testing.assert_array_almost_equal(maxvar, maxvar_s) np.testing.assert_array_almost_equal(vcmt, vcmt_s) for i, j in zip(ifgs, ifgs_mpi): np.testing.assert_array_almost_equal(i.phase_data, j.phase_data) np.testing.assert_array_almost_equal(tsincr, tsincr_mpi, decimal=4) np.testing.assert_array_almost_equal(tscum, tscum_mpi, decimal=4) np.testing.assert_array_almost_equal(rate, rate_mpi, decimal=4) np.testing.assert_array_almost_equal(error, error_mpi, decimal=4) np.testing.assert_array_almost_equal(samples, samples_mpi, decimal=4) # assert linear rate output tifs are same _tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'linrate.tif') _tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'linerror.tif') _tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'linsamples.tif') # assert time series output tifs are same epochlist = algorithm.get_epochs(ifgs)[0] for i in range(tsincr.shape[2]): _tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'tsincr' + '_' + str(epochlist.dates[i + 1]) + ".tif") # 12 timeseries outputs assert i + 1 == tsincr.shape[2] shutil.rmtree(ifgs_mpi_out_dir) # remove mpi out dir shutil.rmtree(params_old[cf.OUT_DIR]) # remove serial out dir
def postprocess_timeseries(rows, cols, params): """ Postprocess time series output. :param rows: xxxx :param cols: xxxx :param params: xxxx :return xxxx """ # pylint: disable=too-many-locals xlks, _, crop = cf.transform_params(params) base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) dest_tifs = cf.get_dest_paths(base_unw_paths, crop, params, xlks) output_dir = params[cf.TMPDIR] # load previously saved prepread_ifgs dict preread_ifgs_file = join(output_dir, 'preread_ifgs.pk') ifgs = cp.load(open(preread_ifgs_file, 'rb')) # metadata and projections gt, md, wkt = ifgs['gt'], ifgs['md'], ifgs['wkt'] epochlist = ifgs['epochlist'] ifgs = [v for v in ifgs.values() if isinstance(v, PrereadIfg)] tiles = run_pyrate.get_tiles(dest_tifs[0], rows, cols) # load the first tsincr file to determine the number of time series tifs tsincr_file = os.path.join(output_dir, 'tsincr_0.npy') tsincr = np.load(file=tsincr_file) # pylint: disable=no-member no_ts_tifs = tsincr.shape[2] # we create 2 x no_ts_tifs as we are splitting tsincr and tscuml # to all processes. process_tifs = mpiops.array_split(range(2 * no_ts_tifs)) # depending on nvelpar, this will not fit in memory # e.g. nvelpar=100, nrows=10000, ncols=10000, 32bit floats need 40GB memory # 32 * 100 * 10000 * 10000 / 8 bytes = 4e10 bytes = 40 GB # the double for loop helps us overcome the memory limit log.info('process {} will write {} ts (incr/cuml) tifs of ' 'total {}'.format(mpiops.rank, len(process_tifs), no_ts_tifs * 2)) for i in process_tifs: tscum_g = np.empty(shape=ifgs[0].shape, dtype=np.float32) if i < no_ts_tifs: for n, t in enumerate(tiles): tscum_file = os.path.join(output_dir, 'tscuml_{}.npy'.format(n)) tscum = np.load(file=tscum_file) md[ifc.EPOCH_DATE] = epochlist.dates[i + 1] # sequence position; first time slice is #0 md['SEQUENCE_POSITION'] = i + 1 tscum_g[t.top_left_y:t.bottom_right_y, t.top_left_x:t.bottom_right_x] = tscum[:, :, i] dest = os.path.join( params[cf.OUT_DIR], 'tscuml' + "_" + str(epochlist.dates[i + 1]) + ".tif") md[ifc.DATA_TYPE] = ifc.CUML shared.write_output_geotiff(md, gt, wkt, tscum_g, dest, np.nan) else: tsincr_g = np.empty(shape=ifgs[0].shape, dtype=np.float32) i %= no_ts_tifs for n, t in enumerate(tiles): tsincr_file = os.path.join(output_dir, 'tsincr_{}.npy'.format(n)) tsincr = np.load(file=tsincr_file) md[ifc.EPOCH_DATE] = epochlist.dates[i + 1] # sequence position; first time slice is #0 md['SEQUENCE_POSITION'] = i + 1 tsincr_g[t.top_left_y:t.bottom_right_y, t.top_left_x:t.bottom_right_x] = tsincr[:, :, i] dest = os.path.join( params[cf.OUT_DIR], 'tsincr' + "_" + str(epochlist.dates[i + 1]) + ".tif") md[ifc.DATA_TYPE] = ifc.INCR shared.write_output_geotiff(md, gt, wkt, tsincr_g, dest, np.nan) log.info('process {} finished writing {} ts (incr/cuml) tifs of ' 'total {}'.format(mpiops.rank, len(process_tifs), no_ts_tifs * 2))
def test_original_ifg_paths(): ifgdir = common.SML_TEST_TIF ifglist_path = join(ifgdir, 'ifms_17') paths = cf.original_ifg_paths(ifglist_path) assert paths[0] == join(ifgdir, 'geo_060619-061002_unw.tif'), str(paths[0]) assert paths[-1] == join(ifgdir, 'geo_070709-070813_unw.tif')