def test_mst_matrix_as_array(self): # Verifies MST matrix func returns array with dict/trees in each cell for i in self.ifgs[3:]: i.phase_data[0, 1] = 0 # partial stack of NODATA to one cell for i in self.ifgs: i.convert_to_nans() # zeros to NaN/NODATA epochs = algorithm.get_epochs(self.ifgs)[0] res = mst._mst_matrix_as_array(self.ifgs) ys, xs = res.shape for y, x in product(range(ys), range(xs)): r = res[y, x] num_nodes = len(r) self.assertTrue(num_nodes < len(epochs.dates)) stack = array([i.phase_data[y, x] for i in self.ifgs]) # 17 ifg stack self.assertTrue( 0 == nsum(stack == 0)) # all 0s should be converted nc = nsum(isnan(stack)) exp_count = len(epochs.dates) - 1 if nc == 0: self.assertEqual(num_nodes, exp_count) elif nc > 5: # rough test: too many nans must reduce the total tree size self.assertTrue(num_nodes <= (17 - nc))
def write_timeseries_geotiff(ifgs, params, tsincr, pr_type): # setup metadata for writing into result files gt, md, wkt = get_geotiff_header_info(ifgs[0].data_path) epochlist = algorithm.get_epochs(ifgs)[0] for i in range(tsincr.shape[2]): md[ifc.EPOCH_DATE] = epochlist.dates[i + 1] md['SEQUENCE_POSITION'] = i + 1 # sequence position data = tsincr[:, :, i] dest = join(params[cf.OUT_DIR], pr_type + "_" + str(epochlist.dates[i + 1]) + ".tif") md[ifc.DATA_TYPE] = pr_type write_output_geotiff(md, gt, wkt, data, dest, np.nan)
def _create_ifg_dict(dest_tifs, params, tiles): """ 1. Convert ifg phase data into numpy binary files. 2. Save the preread_ifgs dict with information about the ifgs that are later used for fast loading of Ifg files in IfgPart class :param list dest_tifs: List of destination tifs :param dict params: Config dictionary :param list tiles: List of all Tile instances :return: preread_ifgs: Dictionary containing information regarding interferograms that are used later in workflow :rtype: dict """ ifgs_dict = {} process_tifs = mpiops.array_split(dest_tifs) shared.save_numpy_phase(dest_tifs, tiles, params) for d in process_tifs: ifg = shared._prep_ifg(d, params) ifgs_dict[d] = PrereadIfg(path=d, nan_fraction=ifg.nan_fraction, master=ifg.master, slave=ifg.slave, time_span=ifg.time_span, nrows=ifg.nrows, ncols=ifg.ncols, metadata=ifg.meta_data) ifg.close() ifgs_dict = _join_dicts(mpiops.comm.allgather(ifgs_dict)) preread_ifgs_file = join(params[cf.TMPDIR], 'preread_ifgs.pk') if mpiops.rank == MASTER_PROCESS: # add some extra information that's also useful later gt, md, wkt = shared.get_geotiff_header_info(process_tifs[0]) ifgs_dict['epochlist'] = algorithm.get_epochs(ifgs_dict)[0] ifgs_dict['gt'] = gt ifgs_dict['md'] = md ifgs_dict['wkt'] = wkt # dump ifgs_dict file for later use cp.dump(ifgs_dict, open(preread_ifgs_file, 'wb')) mpiops.comm.barrier() preread_ifgs = OrderedDict(sorted(cp.load(open(preread_ifgs_file, 'rb')).items())) log.info('Finished converting phase_data to numpy ' 'in process {}'.format(mpiops.rank)) return preread_ifgs
def write_linrate_tifs(ifgs, params, res): # log.info('Writing linrate results') rate, error, samples = res gt, md, wkt = get_geotiff_header_info(ifgs[0].data_path) epochlist = algorithm.get_epochs(ifgs)[0] dest = join(params[cf.OUT_DIR], "linrate.tif") md[ifc.EPOCH_DATE] = epochlist.dates md[ifc.DATA_TYPE] = ifc.LINRATE write_output_geotiff(md, gt, wkt, rate, dest, np.nan) dest = join(params[cf.OUT_DIR], "linerror.tif") md[ifc.DATA_TYPE] = ifc.LINERROR write_output_geotiff(md, gt, wkt, error, dest, np.nan) dest = join(params[cf.OUT_DIR], "linsamples.tif") md[ifc.DATA_TYPE] = ifc.LINSAMP write_output_geotiff(md, gt, wkt, samples, dest, np.nan) write_linrate_numpy_files(error, rate, samples, params)
def _time_series_setup(ifgs, mst, params): """ Convenience function for setting up time series computation parameters """ if len(ifgs) < 1: msg = 'Time series requires 2+ interferograms' raise TimeSeriesError(msg) # if mst is not a single tree then do interpolation interp = 0 if mst_module.mst_from_ifgs(ifgs)[1] else 1 # Parallel Processing parameters parallel = params[cf.PARALLEL] # Time Series parameters tsmethod = params[cf.TIME_SERIES_METHOD] pthresh, smfactor, smorder = _validate_params(params, tsmethod) epochlist = get_epochs(ifgs)[0] nrows = ifgs[0].nrows ncols = ifgs[0].ncols nifgs = len(ifgs) span = diff(epochlist.spans) nepoch = len(epochlist.dates) # epoch number nvelpar = nepoch - 1 # velocity parameters number # nlap = nvelpar - smorder # Laplacian observations number mast_slave_ids = master_slave_ids(epochlist.dates) imaster = [mast_slave_ids[ifg.master] for ifg in ifgs] islave = [mast_slave_ids[ifg.slave] for ifg in ifgs] imaster = min(imaster, islave) islave = max(imaster, islave) b0_mat = zeros((nifgs, nvelpar)) for i in range(nifgs): b0_mat[i, imaster[i]:islave[i]] = span[imaster[i]:islave[i]] # change the sign if slave is earlier than master isign = where(imaster > islave) b0_mat[isign[0], :] = -b0_mat[isign[0], :] tsvel_matrix = np.empty(shape=(nrows, ncols, nvelpar), dtype=float32) ifg_data = np.zeros((nifgs, nrows, ncols), dtype=float32) for ifg_num in range(nifgs): ifg_data[ifg_num] = ifgs[ifg_num].phase_data if mst is None: mst = ~isnan(ifg_data) return b0_mat, interp, pthresh, smfactor, smorder, tsmethod, ifg_data, \ mst, ncols, nrows, nvelpar, parallel, span, tsvel_matrix
def _ts_to_ifgs(tsincr, preread_ifgs): """ Function that converts an incremental displacement time series into interferometric phase observations. Used to re-construct an interferogram network from a time series. :param ndarray tsincr: incremental time series array of size (ifg.shape, nepochs-1) :param dict preread_ifgs: Dictionary of shared.PrereadIfg class instances :return: None, interferograms are saved to disk """ log.info('Converting time series to ifgs') ifgs = list(OrderedDict(sorted(preread_ifgs.items())).values()) _, n = get_epochs(ifgs) index_master, index_slave = n[:len(ifgs)], n[len(ifgs):] for i, ifg in enumerate(ifgs): phase = np.sum(tsincr[:, :, index_master[i]:index_slave[i]], axis=2) _save_aps_corrected_phase(ifg.path, phase)
def get_nml(ifg_list_instance, nodata_value, nan_conversion=False): """ A reproduction of getnml.m, the function in Matlab Pirate. Note: the Matlab version tested does not have nan's. replaces the ifg_list_instance in place. :param ifg_list_instance: xxxx :param nodata_value: xxxx :param nan_conversion: xxxx :return xxxxx """ _epoch_list, n = get_epochs(ifg_list_instance.ifgs) ifg_list_instance.reshape_n(n) if nan_conversion: ifg_list_instance.update_nan_frac(nodata_value) # turn on for nan conversion ifg_list_instance.convert_nans(nan_conversion=nan_conversion) ifg_list_instance.make_data_stack() return ifg_list_instance, _epoch_list
def get_nml(ifg_list_instance, nodata_value, nan_conversion=False): """ A translation of the Matlab Pirate 'getnml.m' function. Note: the Matlab version tested does not have nan's. :param xxx(eg str, tuple, int, float...) ifg_list_instance: xxxx :param float nodata_value: No data value in image :param bool nan_conversion: Convert NaNs :return: ifg_list_instance: replaces in place :rtype: list :return: _epoch_list: list of epochs :rtype: list """ _epoch_list, n = algorithm.get_epochs(ifg_list_instance.ifgs) ifg_list_instance.reshape_n(n) if nan_conversion: ifg_list_instance.update_nan_frac(nodata_value) # turn on for nan conversion ifg_list_instance.convert_nans(nan_conversion=nan_conversion) ifg_list_instance.make_data_stack() return ifg_list_instance, _epoch_list
def test_get_epochs(self): def str2date(s): segs = s[:4], s[4:6], s[6:] # year, month, day return date(*[int(sg) for sg in segs]) raw_date = ['20060619', '20060828', '20061002', '20061106', '20061211', '20070115', '20070219', '20070326', '20070430', '20070604', '20070709', '20070813', '20070917'] exp_dates = [str2date(d) for d in raw_date] exp_repeat = [1, 1, 3, 3, 4, 3, 3, 3, 3, 3, 3, 2, 2] exp_spans = [0, 0.1916, 0.2875, 0.3833, 0.4791, 0.5749, 0.6708, 0.7666, 0.8624, 0.9582, 1.0541, 1.1499, 1.2457] ifms = join(SML_TEST_TIF, "ifms_17") ifgs = [Ifg(join(SML_TEST_TIF, p)) for p in parse_namelist(ifms)] for i in ifgs: i.open() epochs = get_epochs(ifgs)[0] self.assertTrue((exp_dates == epochs.dates).all()) self.assertTrue((exp_repeat == epochs.repeat).all()) assert_array_almost_equal(exp_spans, epochs.spans, decimal=4)
def test_timeseries_linrate_mpi(mpisync, tempdir, modify_config, ref_est_method, row_splits, col_splits, get_crop, orbfit_lks, orbfit_method, orbfit_degrees): params = modify_config outdir = mpiops.run_once(tempdir) params[cf.OUT_DIR] = outdir params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR) params[cf.DEM_HEADER_FILE] = SML_TEST_DEM_HDR_GAMMA params[cf.REF_EST_METHOD] = ref_est_method params[cf.IFG_CROP_OPT] = get_crop params[cf.ORBITAL_FIT_LOOKS_Y] = orbfit_lks params[cf.ORBITAL_FIT_LOOKS_X] = orbfit_lks params[cf.ORBITAL_FIT_METHOD] = orbfit_method params[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees xlks, ylks, crop = cf.transform_params(params) if xlks * col_splits > 45 or ylks * row_splits > 70: print('skipping test because lks and col_splits are not compatible') return # skip some tests in travis to run CI faster if TRAVIS and (xlks % 2 or row_splits % 2 or col_splits % 2 or orbfit_lks % 2): print('Skipping in travis env for faster CI run') return print("xlks={}, ref_est_method={}, row_splits={}, col_splits={}, " "get_crop={}, orbfit_lks={}, orbfit_method={}, " "rank={}".format(xlks, ref_est_method, row_splits, col_splits, get_crop, orbfit_lks, orbfit_method, orbfit_degrees, mpiops.rank)) base_unw_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST]) # dest_paths are tifs that have been geotif converted and multilooked dest_paths = cf.get_dest_paths(base_unw_paths, crop, params, xlks) # run prepifg, create the dest_paths files if mpiops.rank == 0: run_prepifg.gamma_prepifg(base_unw_paths, params) mpiops.comm.barrier() (refpx, refpy), maxvar, vcmt = run_pyrate.process_ifgs(ifg_paths=dest_paths, params=params, rows=row_splits, cols=col_splits) tiles = mpiops.run_once(run_pyrate.get_tiles, dest_paths[0], rows=row_splits, cols=col_splits) postprocessing.postprocess_linrate(row_splits, col_splits, params) postprocessing.postprocess_timeseries(row_splits, col_splits, params) ifgs_mpi_out_dir = params[cf.OUT_DIR] ifgs_mpi = small_data_setup(datafiles=dest_paths) # single process timeseries/linrate calculation if mpiops.rank == 0: params_old = modify_config params_old[cf.OUT_DIR] = tempdir() params_old[cf.REF_EST_METHOD] = ref_est_method params_old[cf.IFG_CROP_OPT] = get_crop params_old[cf.ORBITAL_FIT_LOOKS_Y] = orbfit_lks params_old[cf.ORBITAL_FIT_LOOKS_X] = orbfit_lks params_old[cf.ORBITAL_FIT_METHOD] = orbfit_method params_old[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees xlks, ylks, crop = cf.transform_params(params_old) base_unw_paths = cf.original_ifg_paths(params_old[cf.IFG_FILE_LIST]) dest_paths = cf.get_dest_paths(base_unw_paths, crop, params_old, xlks) run_prepifg.gamma_prepifg(base_unw_paths, params_old) ifgs = shared.pre_prepare_ifgs(dest_paths, params_old) mst_grid = tests.common.mst_calculation(dest_paths, params_old) refy, refx = refpixel.ref_pixel(ifgs, params_old) assert (refx == refpx) and (refy == refpy) # both must match pyrate.orbital.remove_orbital_error(ifgs, params_old) ifgs = common.prepare_ifgs_without_phase(dest_paths, params_old) rpe.estimate_ref_phase(ifgs, params_old, refx, refy) ifgs = shared.pre_prepare_ifgs(dest_paths, params_old) maxvar_s = [vcm.cvd(i, params_old)[0] for i in ifgs] vcmt_s = vcm.get_vcmt(ifgs, maxvar) tsincr, tscum, _ = tests.common.compute_time_series( ifgs, mst_grid, params, vcmt) rate, error, samples = tests.common.calculate_linear_rate( ifgs, params_old, vcmt, mst_grid) mst_mpi = reconstruct_mst(ifgs[0].shape, tiles, params[cf.TMPDIR]) np.testing.assert_array_almost_equal(mst_grid, mst_mpi) tsincr_mpi, tscum_mpi = reconstruct_times_series( ifgs[0].shape, tiles, params[cf.TMPDIR]) rate_mpi, error_mpi, samples_mpi = \ [reconstruct_linrate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in ['linrate', 'linerror', 'linsamples']] np.testing.assert_array_almost_equal(maxvar, maxvar_s) np.testing.assert_array_almost_equal(vcmt, vcmt_s) for i, j in zip(ifgs, ifgs_mpi): np.testing.assert_array_almost_equal(i.phase_data, j.phase_data) np.testing.assert_array_almost_equal(tsincr, tsincr_mpi, decimal=4) np.testing.assert_array_almost_equal(tscum, tscum_mpi, decimal=4) np.testing.assert_array_almost_equal(rate, rate_mpi, decimal=4) np.testing.assert_array_almost_equal(error, error_mpi, decimal=4) np.testing.assert_array_almost_equal(samples, samples_mpi, decimal=4) # assert linear rate output tifs are same _tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'linrate.tif') _tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'linerror.tif') _tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'linsamples.tif') # assert time series output tifs are same epochlist = algorithm.get_epochs(ifgs)[0] for i in range(tsincr.shape[2]): _tifs_same(ifgs_mpi_out_dir, params_old[cf.OUT_DIR], 'tsincr' + '_' + str(epochlist.dates[i + 1]) + ".tif") # 12 timeseries outputs assert i + 1 == tsincr.shape[2] shutil.rmtree(ifgs_mpi_out_dir) # remove mpi out dir shutil.rmtree(params_old[cf.OUT_DIR]) # remove serial out dir
# tsincr matrix from matlab using svd timeseries method tsincr_svd = sio.loadmat( os.path.join(SML_TEST_DIR, 'matlab_aps', 'tsincr_svd.mat')) # prepread ifgs pickle file preread_pk = 'preread_ifgs.pk' if PY3 else 'preread_ifgs_py2.pk' ifgs_pk = os.path.join(SML_TEST_DIR, 'matlab_aps', preread_pk) ifgs_pk = pickle.load(open(ifgs_pk, 'rb')) # tlp filter output from matlab ts_hp = sio.loadmat(os.path.join(SML_TEST_DIR, 'matlab_aps', 'ts_hp.mat')) ts_hp_m2 = sio.loadmat(os.path.join(SML_TEST_DIR, 'matlab_aps', 'ts_hp_m2.mat')) ts_hp_m3 = sio.loadmat(os.path.join(SML_TEST_DIR, 'matlab_aps', 'ts_hp_m3.mat')) epochlist = get_epochs(ifgs_pk)[0] params = cf.get_config_params(os.path.join(TEST_CONF_GAMMA)) @pytest.fixture(params=[1, 2, 3]) def tlpfilter_method(request): return request.param @pytest.fixture(params=[1, 2]) def slpfilter_method(request): return request.param def test_tlpfilter_matlab(tlpfilter_method): params[cf.TLPF_METHOD] = tlpfilter_method
def time_series_setup(ifgs, mst, params): """ Convenience function setting up time series computation parameters. :param ifgs: xxxx :param mst: xxxx :param params: xxxx :return xxxx """ if len(ifgs) < 1: msg = 'Time series requires 2+ interferograms' raise TimeSeriesError(msg) # if mst is not a single tree then do interpolation interp = 0 if mst_module.mst_from_ifgs(ifgs)[1] else 1 # Parallel Processing parameters parallel = params[cf.PARALLEL] # Time Series parameters tsmethod = params[cf.TIME_SERIES_METHOD] if tsmethod == 1 and params[cf.TIME_SERIES_SM_ORDER] is None: missing_option_error(cf.TIME_SERIES_SM_ORDER) else: smorder = params[cf.TIME_SERIES_SM_ORDER] if tsmethod == 1 and params[cf.TIME_SERIES_SM_FACTOR] is None: missing_option_error(cf.TIME_SERIES_SM_FACTOR) else: smfactor = np.power(10, params[cf.TIME_SERIES_SM_FACTOR]) if params[cf.TIME_SERIES_PTHRESH] is None: missing_option_error(cf.TIME_SERIES_PTHRESH) else: pthresh = params[cf.TIME_SERIES_PTHRESH] if pthresh < 0.0 or pthresh > 1000: raise ValueError( "minimum number of coherent observations for a pixel" " TIME_SERIES_PTHRESH setting must be >= 0.0 and <= 1000") epochlist = get_epochs(ifgs)[0] nrows = ifgs[0].nrows ncols = ifgs[0].ncols nifgs = len(ifgs) span = diff(epochlist.spans) nepoch = len(epochlist.dates) # epoch number nvelpar = nepoch - 1 # velocity parameters number # nlap = nvelpar - smorder # Laplacian observations number mast_slave_ids = master_slave_ids(epochlist.dates) imaster = [mast_slave_ids[ifg.master] for ifg in ifgs] islave = [mast_slave_ids[ifg.slave] for ifg in ifgs] imaster = min(imaster, islave) islave = max(imaster, islave) b0_mat = zeros((nifgs, nvelpar)) for i in range(nifgs): b0_mat[i, imaster[i]:islave[i]] = span[imaster[i]:islave[i]] # change the sign if slave is earlier than master isign = where(imaster > islave) b0_mat[isign[0], :] = -b0_mat[isign[0], :] tsvel_matrix = np.empty(shape=(nrows, ncols, nvelpar), dtype=float32) ifg_data = np.zeros((nifgs, nrows, ncols), dtype=float32) for ifg_num in range(nifgs): ifg_data[ifg_num] = ifgs[ifg_num].phase_data if mst is None: mst = ~isnan(ifg_data) return b0_mat, interp, pthresh, smfactor, smorder, tsmethod, ifg_data, \ mst, ncols, nrows, nvelpar, parallel, span, tsvel_matrix