def __save_ifgs_dict_with_headers_and_epochs(dest_tifs, ifgs_dict, params, process_tifs): tmpdir = params[cf.TMPDIR] if not os.path.exists(tmpdir): shared.mkdir_p(tmpdir) preread_ifgs_file = Configuration.preread_ifgs(params) nifgs = len(dest_tifs) # add some extra information that's also useful later gt, md, wkt = shared.get_geotiff_header_info( process_tifs[0].tmp_sampled_path) epochlist = algorithm.get_epochs(ifgs_dict)[0] log.info('Found {} unique epochs in the {} interferogram network'.format( len(epochlist.dates), nifgs)) ifgs_dict['epochlist'] = epochlist ifgs_dict['gt'] = gt ifgs_dict['md'] = md ifgs_dict['wkt'] = wkt # dump ifgs_dict file for later use cp.dump(ifgs_dict, open(preread_ifgs_file, 'wb')) for k in ['gt', 'epochlist', 'md', 'wkt']: ifgs_dict.pop(k) return ifgs_dict
def test_get_epochs(self): def str2date(s): segs = s[:4], s[4:6], s[6:] # year, month, day return date(*[int(sg) for sg in segs]) raw_date = [ '20060619', '20060828', '20061002', '20061106', '20061211', '20070115', '20070219', '20070326', '20070430', '20070604', '20070709', '20070813', '20070917' ] exp_dates = [str2date(d) for d in raw_date] exp_repeat = [1, 1, 3, 3, 4, 3, 3, 3, 3, 3, 3, 2, 2] exp_spans = [ 0, 0.1916, 0.2875, 0.3833, 0.4791, 0.5749, 0.6708, 0.7666, 0.8624, 0.9582, 1.0541, 1.1499, 1.2457 ] ifms = join(SML_TEST_TIF, "ifms_17") ifgs = [Ifg(join(SML_TEST_TIF, p)) for p in parse_namelist(ifms)] for i in ifgs: i.open() epochs = get_epochs(ifgs)[0] self.assertTrue((exp_dates == epochs.dates).all()) self.assertTrue((exp_repeat == epochs.repeat).all()) assert_array_almost_equal(exp_spans, epochs.spans, decimal=4)
def test_mst_matrix_as_array(self): # Verifies MST matrix func returns array with dict/trees in each cell for i in self.ifgs[3:]: i.phase_data[0, 1] = 0 # partial stack of NODATA to one cell for i in self.ifgs: i.convert_to_nans() # zeros to NaN/NODATA epochs = algorithm.get_epochs(self.ifgs)[0] res = mst._mst_matrix_as_array(self.ifgs) ys, xs = res.shape for y, x in product(range(ys), range(xs)): r = res[y, x] num_nodes = len(r) self.assertTrue(num_nodes < len(epochs.dates)) stack = array([i.phase_data[y, x] for i in self.ifgs]) # 17 ifg stack self.assertTrue( 0 == nsum(stack == 0)) # all 0s should be converted nc = nsum(isnan(stack)) exp_count = len(epochs.dates) - 1 if nc == 0: self.assertEqual(num_nodes, exp_count) elif nc > 5: # rough test: too many nans must reduce the total tree size self.assertTrue(num_nodes <= (17 - nc))
def _create_ifg_dict(dest_tifs, params): """ 1. Convert ifg phase data into numpy binary files. 2. Save the preread_ifgs dict with information about the ifgs that are later used for fast loading of Ifg files in IfgPart class :param list dest_tifs: List of destination tifs :param dict params: Config dictionary :param list tiles: List of all Tile instances :return: preread_ifgs: Dictionary containing information regarding interferograms that are used later in workflow :rtype: dict """ ifgs_dict = {} nifgs = len(dest_tifs) process_tifs = mpiops.array_split(dest_tifs) for d in process_tifs: ifg = shared._prep_ifg(d, params) ifgs_dict[d] = PrereadIfg(path=d, nan_fraction=ifg.nan_fraction, master=ifg.master, slave=ifg.slave, time_span=ifg.time_span, nrows=ifg.nrows, ncols=ifg.ncols, metadata=ifg.meta_data) ifg.close() ifgs_dict = _join_dicts(mpiops.comm.allgather(ifgs_dict)) preread_ifgs_file = join(params[cf.TMPDIR], 'preread_ifgs.pk') if mpiops.rank == MASTER_PROCESS: # add some extra information that's also useful later gt, md, wkt = shared.get_geotiff_header_info(process_tifs[0]) epochlist = algorithm.get_epochs(ifgs_dict)[0] log.info( 'Found {} unique epochs in the {} interferogram network'.format( len(epochlist.dates), nifgs)) ifgs_dict['epochlist'] = epochlist ifgs_dict['gt'] = gt ifgs_dict['md'] = md ifgs_dict['wkt'] = wkt # dump ifgs_dict file for later use cp.dump(ifgs_dict, open(preread_ifgs_file, 'wb')) mpiops.comm.barrier() preread_ifgs = OrderedDict( sorted(cp.load(open(preread_ifgs_file, 'rb')).items())) log.debug('Finished converting phase_data to numpy in process {}'.format( mpiops.rank)) return preread_ifgs
def write_timeseries_geotiff(ifgs, params, tsincr, pr_type): # setup metadata for writing into result files gt, md, wkt = get_geotiff_header_info(ifgs[0].data_path) epochlist = algorithm.get_epochs(ifgs)[0] for i in range(tsincr.shape[2]): md[ifc.EPOCH_DATE] = epochlist.dates[i + 1] md['SEQUENCE_POSITION'] = i + 1 # sequence position data = tsincr[:, :, i] dest = join(params[cf.OUT_DIR], pr_type + "_" + str(epochlist.dates[i + 1]) + ".tif") md[ifc.DATA_TYPE] = pr_type write_output_geotiff(md, gt, wkt, data, dest, np.nan)
def linear_rate_array(tscuml, ifgs, params): """ This function loops over all pixels in a 3-dimensional cumulative time series array and calculates the linear rate (line of best fit) for each pixel using linear regression. :param ndarray tscuml: 3-dimensional cumulative time series array :param list ifgs: list of interferogram class objects. :param dict params: Configuration parameters :return: linrate: Linear rate map from linear regression :rtype: ndarray :return: intercept: Array of Y-axis intercepts from linear regression :rtype: ndarray :return: rsquared: Array of R-squared value of the linear regression :rtype: ndarray :return: error: Array of standard errors of the linear regression :rtype: ndarray :return: samples: Number of observations used in linear regression for each pixel :rtype: ndarray """ b0_mat, interp, p_thresh, sm_factor, sm_order, ts_method, ifg_data, mst, \ ncols, nrows, nvelpar, span, tsvel_matrix = \ _time_series_setup(ifgs, params) epochlist = get_epochs(ifgs)[0] # get cumulative time per epoch t = asarray(epochlist.spans) # test for equal length of input vectors if tscuml.shape[2] != len(t): raise TimeSeriesError( "linear_rate_array: tscuml and nepochs are not equal length") # preallocate empty arrays for results linrate = np.empty([nrows, ncols], dtype=float32) rsquared = np.empty([nrows, ncols], dtype=float32) error = np.empty([nrows, ncols], dtype=float32) intercept = np.empty([nrows, ncols], dtype=np.float32) samples = np.empty([nrows, ncols], dtype=np.float32) # pixel-by-pixel calculation. # nested loops to loop over the 2 image dimensions for i in range(nrows): for j in range(ncols): linrate[i, j], intercept[i, j], rsquared[i, j], error[i, j], samples[i, j] = \ linear_rate_pixel(tscuml[i, j, :], t) return linrate, intercept, rsquared, error, samples
def write_stackrate_tifs(ifgs, params, res): rate, error, samples = res gt, md, wkt = get_geotiff_header_info(ifgs[0].data_path) epochlist = algorithm.get_epochs(ifgs)[0] dest = join(params[cf.OUT_DIR], "stack_rate.tif") md[ifc.EPOCH_DATE] = epochlist.dates md[ifc.DATA_TYPE] = ifc.STACKRATE write_output_geotiff(md, gt, wkt, rate, dest, np.nan) dest = join(params[cf.OUT_DIR], "stack_error.tif") md[ifc.DATA_TYPE] = ifc.STACKERROR write_output_geotiff(md, gt, wkt, error, dest, np.nan) dest = join(params[cf.OUT_DIR], "stack_samples.tif") md[ifc.DATA_TYPE] = ifc.STACKSAMP write_output_geotiff(md, gt, wkt, samples, dest, np.nan) write_stackrate_numpy_files(error, rate, samples, params)
def write_linrate_tifs(ifgs, params, res): # log.info('Writing linrate results') rate, error, samples = res gt, md, wkt = get_geotiff_header_info(ifgs[0].data_path) epochlist = algorithm.get_epochs(ifgs)[0] dest = join(params[cf.OUT_DIR], "linrate.tif") md[ifc.EPOCH_DATE] = epochlist.dates md[ifc.DATA_TYPE] = ifc.LINRATE write_output_geotiff(md, gt, wkt, rate, dest, np.nan) dest = join(params[cf.OUT_DIR], "linerror.tif") md[ifc.DATA_TYPE] = ifc.LINERROR write_output_geotiff(md, gt, wkt, error, dest, np.nan) dest = join(params[cf.OUT_DIR], "linsamples.tif") md[ifc.DATA_TYPE] = ifc.LINSAMP write_output_geotiff(md, gt, wkt, samples, dest, np.nan) write_linrate_numpy_files(error, rate, samples, params)
def _time_series_setup(ifgs, mst, params): """ Convenience function for setting up time series computation parameters """ if len(ifgs) < 1: msg = 'Time series requires 2+ interferograms' raise TimeSeriesError(msg) # if mst is not a single tree then do interpolation interp = 0 if mst_module.mst_from_ifgs(ifgs)[1] else 1 # Parallel Processing parameters parallel = params[cf.PARALLEL] # Time Series parameters tsmethod = params[cf.TIME_SERIES_METHOD] pthresh, smfactor, smorder = _validate_params(params, tsmethod) epochlist = get_epochs(ifgs)[0] nrows = ifgs[0].nrows ncols = ifgs[0].ncols nifgs = len(ifgs) span = diff(epochlist.spans) nepoch = len(epochlist.dates) # epoch number nvelpar = nepoch - 1 # velocity parameters number # nlap = nvelpar - smorder # Laplacian observations number mast_slave_ids = master_slave_ids(epochlist.dates) imaster = [mast_slave_ids[ifg.master] for ifg in ifgs] islave = [mast_slave_ids[ifg.slave] for ifg in ifgs] imaster = min(imaster, islave) islave = max(imaster, islave) b0_mat = zeros((nifgs, nvelpar)) for i in range(nifgs): b0_mat[i, imaster[i]:islave[i]] = span[imaster[i]:islave[i]] # change the sign if slave is earlier than master isign = where(np.atleast_1d(imaster) > np.atleast_1d(islave)) b0_mat[isign[0], :] = -b0_mat[isign[0], :] tsvel_matrix = np.empty(shape=(nrows, ncols, nvelpar), dtype=float32) ifg_data = np.zeros((nifgs, nrows, ncols), dtype=float32) for ifg_num in range(nifgs): ifg_data[ifg_num] = ifgs[ifg_num].phase_data if mst is None: mst = ~isnan(ifg_data) return b0_mat, interp, pthresh, smfactor, smorder, tsmethod, ifg_data, \ mst, ncols, nrows, nvelpar, parallel, span, tsvel_matrix
def get_nml(ifg_list_instance, nodata_value, nan_conversion=False): """ :param xxx(eg str, tuple, int, float...) ifg_list_instance: xxxx :param float nodata_value: No data value in image :param bool nan_conversion: Convert NaNs :return: ifg_list_instance: replaces in place :rtype: list :return: _epoch_list: list of epochs :rtype: list """ _epoch_list, n = algorithm.get_epochs(ifg_list_instance.ifgs) ifg_list_instance.reshape_n(n) if nan_conversion: ifg_list_instance.update_nan_frac(nodata_value) # turn on for nan conversion ifg_list_instance.convert_nans(nan_conversion=nan_conversion) ifg_list_instance.make_data_stack() return ifg_list_instance, _epoch_list
def _ts_to_ifgs(tsincr, preread_ifgs): """ Function that converts an incremental displacement time series into interferometric phase observations. Used to re-construct an interferogram network from a time series. :param ndarray tsincr: incremental time series array of size (ifg.shape, nepochs-1) :param dict preread_ifgs: Dictionary of shared.PrereadIfg class instances :return: None, interferograms are saved to disk """ log.debug('Reconstructing interferometric observations from time series') ifgs = list(OrderedDict(sorted(preread_ifgs.items())).values()) _, n = get_epochs(ifgs) index_master, index_slave = n[:len(ifgs)], n[len(ifgs):] for i, ifg in enumerate(ifgs): phase = np.sum(tsincr[:, :, index_master[i]:index_slave[i]], axis=2) _save_aps_corrected_phase(ifg.path, phase)