def test_first_second_ids(self): d0 = date(2006, 6, 19) d1 = date(2006, 8, 28) d2 = date(2006, 10, 2) d3 = date(2006, 11, 6) exp = {d0: 0, d1: 1, d2: 2, d3: 3} # test unordered and with duplicates self.assertEqual(exp, first_second_ids([d3, d0, d2, d1])) self.assertEqual(exp, first_second_ids([d3, d0, d2, d1, d3, d0]))
def get_date_ids(ifgs): ''' Returns unique epoch date IDs from the given Ifgs. ''' dates = [] for ifg in ifgs: dates += [ifg.first, ifg.second] return first_second_ids(dates)
def get_network_design_matrix(ifgs, degree, scale, intercept=True): # pylint: disable=too-many-locals """ Returns larger-format design matrix for network error correction. The network design matrix includes rows which relate to those of NaN cells. :param list ifgs: List of Ifg class objects :param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC) :param int scale: Scale factor for design matrix to improve inversion robustness :param bool intercept: whether to include columns for intercept estimation. :return: netdm: network design matrix :rtype: ndarray """ if degree not in [PLANAR, QUADRATIC, PART_CUBIC]: raise OrbitalError("Invalid degree argument") if scale < 1: raise OrbitalError("Scale argument must be greater or equal to 1") nifgs = len(ifgs) if nifgs < 1: # can feasibly do correction on a single Ifg/2 epochs raise OrbitalError("Invalid number of Ifgs: %s" % nifgs) # init sparse network design matrix nepochs = len(set(get_all_epochs(ifgs))) # no intercepts here; they are included separately below ncoef = _get_num_params(degree) shape = [ifgs[0].num_cells * nifgs, ncoef * nepochs] if intercept: shape[1] += nepochs # add extra space for intercepts netdm = zeros(shape, dtype=float32) # calc location for individual design matrices dates = [ifg.first for ifg in ifgs] + [ifg.second for ifg in ifgs] ids = first_second_ids(dates) tmpdm = get_design_matrix(ifgs[0], degree, intercept=intercept, scale=scale) # iteratively build up sparse matrix for i, ifg in enumerate(ifgs): rs = i * ifg.num_cells # starting row m = ids[ifg.first] * (ncoef + intercept) # start col for first s = ids[ifg.second] * (ncoef + intercept) # start col for second netdm[rs:rs + ifg.num_cells, m:m + ncoef + intercept] = -tmpdm netdm[rs:rs + ifg.num_cells, s:s + ncoef + intercept] = tmpdm return netdm
def get_network_design_matrix(ifgs, degree, offset): # pylint: disable=too-many-locals """ Returns larger-format design matrix for network error correction. The network design matrix includes rows which relate to those of NaN cells. :param list ifgs: List of Ifg class objects :param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC) :param bool offset: True to include offset cols, otherwise False. :return: netdm: network design matrix :rtype: ndarray """ if degree not in [PLANAR, QUADRATIC, PART_CUBIC]: raise OrbitalError("Invalid degree argument") nifgs = len(ifgs) if nifgs < 1: # can feasibly do correction on a single Ifg/2 epochs raise OrbitalError("Invalid number of Ifgs: %s" % nifgs) # init sparse network design matrix nepochs = len(set(get_all_epochs(ifgs))) # no offsets: they are made separately below ncoef = _get_num_params(degree) shape = [ifgs[0].num_cells * nifgs, ncoef * nepochs] if offset: shape[1] += nifgs # add extra block for offset cols netdm = zeros(shape, dtype=float32) # calc location for individual design matrices dates = [ifg.first for ifg in ifgs] + [ifg.second for ifg in ifgs] ids = first_second_ids(dates) offset_col = nepochs * ncoef # base offset for the offset cols tmpdm = get_design_matrix(ifgs[0], degree, offset=False) # iteratively build up sparse matrix for i, ifg in enumerate(ifgs): rs = i * ifg.num_cells # starting row m = ids[ifg.first] * ncoef # start col for first s = ids[ifg.second] * ncoef # start col for second netdm[rs:rs + ifg.num_cells, m:m + ncoef] = -tmpdm netdm[rs:rs + ifg.num_cells, s:s + ncoef] = tmpdm # offsets are diagonal cols across the extra array block created above if offset: netdm[rs:rs + ifg.num_cells, offset_col + i] = 1 # init offset cols return netdm
def get_vcmt(ifgs, maxvar): """ Assembles a temporal variance/covariance matrix using the method described by Biggs et al., Geophys. J. Int, 2007. Matrix elements are evaluated according to sig_i * sig_j * C_ij where i and j are two interferograms and C is a matrix of coefficients: C = 1 if the first and second epochs of i and j are equal C = 0.5 if have i and j share either a common first or second epoch C = -0.5 if the first of i or j equals the second of the other C = 0 otherwise :param list ifgs: A list of pyrate.shared.Ifg class objects. :param ndarray maxvar: numpy array of maximum variance values for the interferograms. :return: vcm_t: temporal variance-covariance matrix :rtype: ndarray """ # pylint: disable=too-many-locals # c=0.5 for common first or second; c=-0.5 if first # of one matches second of another if isinstance(ifgs, dict): ifgs = {k: v for k, v in ifgs.items() if isinstance(v, PrereadIfg)} ifgs = OrderedDict(sorted(ifgs.items())) # pylint: disable=redefined-variable-type ifgs = ifgs.values() nifgs = len(ifgs) vcm_pat = zeros((nifgs, nifgs)) dates = [ifg.first for ifg in ifgs] + [ifg.second for ifg in ifgs] ids = first_second_ids(dates) for i, ifg in enumerate(ifgs): mas1, slv1 = ids[ifg.first], ids[ifg.second] for j, ifg2 in enumerate(ifgs): mas2, slv2 = ids[ifg2.first], ids[ifg2.second] if mas1 == mas2 or slv1 == slv2: vcm_pat[i, j] = 0.5 if mas1 == slv2 or slv1 == mas2: vcm_pat[i, j] = -0.5 if mas1 == mas2 and slv1 == slv2: vcm_pat[i, j] = 1.0 # diagonal elements # make covariance matrix in time domain std = sqrt(maxvar).reshape((nifgs, 1)) vcm_t = std * std.transpose() return vcm_t * vcm_pat
def _time_series_setup(ifgs, params, mst=None): """ Convenience function for setting up time series computation parameters """ if len(ifgs) < 1: msg = 'Time series requires 2+ interferograms' raise TimeSeriesError(msg) # if mst is not a single tree then do interpolation interp = 0 if mst_module.mst_from_ifgs(ifgs)[1] else 1 # Time Series parameters tsmethod = params[C.TIME_SERIES_METHOD] pthresh, smfactor, smorder = _validate_params(params, tsmethod) epochlist = get_epochs(ifgs)[0] nrows = ifgs[0].nrows ncols = ifgs[0].ncols nifgs = len(ifgs) span = diff(epochlist.spans) nepoch = len(epochlist.dates) # epoch number nvelpar = nepoch - 1 # velocity parameters number # nlap = nvelpar - smorder # Laplacian observations number mast_second_ids = first_second_ids(epochlist.dates) ifirst = [mast_second_ids[ifg.first] for ifg in ifgs] isecond = [mast_second_ids[ifg.second] for ifg in ifgs] ifirst = min(ifirst, isecond) isecond = max(ifirst, isecond) b0_mat = zeros((nifgs, nvelpar)) for i in range(nifgs): b0_mat[i, ifirst[i]:isecond[i]] = span[ifirst[i]:isecond[i]] # change the sign if second is earlier than first isign = where(np.atleast_1d(ifirst) > np.atleast_1d(isecond)) b0_mat[isign[0], :] = -b0_mat[isign[0], :] tsvel_matrix = np.empty(shape=(nrows, ncols, nvelpar), dtype=float32) ifg_data = np.zeros((nifgs, nrows, ncols), dtype=float32) for ifg_num in range(nifgs): ifg_data[ifg_num] = ifgs[ifg_num].phase_data if mst is None: mst = ~isnan(ifg_data) return b0_mat, interp, pthresh, smfactor, smorder, tsmethod, ifg_data, \ mst, ncols, nrows, nvelpar, span, tsvel_matrix
def network_orbital_correction(ifg_paths, params, m_ifgs: Optional[List] = None): """ This algorithm implements a network inversion to determine orbital corrections for a set of interferograms forming a connected network. Warning: This will write orbital error corrected phase_data to the ifgs. :param list ifg_paths: List of Ifg class objects reduced to a minimum spanning tree network :param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC) :param bool offset: True to calculate the model using offsets :param dict params: dictionary of configuration parameters :param list m_ifgs: list of multilooked Ifg class objects (sequence must be multilooked versions of 'ifgs' arg) :param dict preread_ifgs: Dictionary containing information specifically for MPI jobs (optional) :return: None - interferogram phase data is updated and saved to disk """ # pylint: disable=too-many-locals, too-many-arguments offset = params[cf.ORBFIT_OFFSET] degree = params[cf.ORBITAL_FIT_DEGREE] preread_ifgs = params[cf.PREREAD_IFGS] # all orbit corrections available? if isinstance(ifg_paths[0], str): if __check_and_apply_orberrors_found_on_disc(ifg_paths, params): log.warning("Reusing orbfit errors from previous run!!!") return # all corrections are available in numpy files already saved - return ifgs = [shared.Ifg(i) for i in ifg_paths] else: # alternate test paths # TODO: improve ifgs = ifg_paths src_ifgs = ifgs if m_ifgs is None else m_ifgs src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst vphase = vstack([i.phase_data.reshape((i.num_cells, 1)) for i in src_ifgs]) vphase = squeeze(vphase) B = get_network_design_matrix(src_ifgs, degree, offset) # filter NaNs out before getting model B = B[~isnan(vphase)] orbparams = dot(pinv(B, 1e-6), vphase[~isnan(vphase)]) ncoef = _get_num_params(degree) if preread_ifgs: temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values() ids = first_second_ids(get_all_epochs(temp_ifgs)) else: ids = first_second_ids(get_all_epochs(ifgs)) coefs = [ orbparams[i:i + ncoef] for i in range(0, len(set(ids)) * ncoef, ncoef) ] # create full res DM to expand determined coefficients into full res # orbital correction (eg. expand coarser model to full size) if preread_ifgs: temp_ifg = Ifg(ifg_paths[0]) # ifgs here are paths temp_ifg.open() dm = get_design_matrix(temp_ifg, degree, offset=False) temp_ifg.close() else: ifg = ifgs[0] dm = get_design_matrix(ifg, degree, offset=False) for i in ifg_paths: # open if not Ifg instance if isinstance(i, str): # pragma: no cover # are paths i = Ifg(i) i.open(readonly=False) shared.nan_and_mm_convert(i, params) _remove_network_orb_error(coefs, dm, i, ids, offset, params)
def network_orbital_correction(ifg_paths, params, m_ifgs: Optional[List] = None): """ This algorithm implements a network inversion to determine orbital corrections for a set of interferograms forming a connected network. Warning: This will write orbital error corrected phase_data to the ifgs. :param list ifg_paths: List of Ifg class objects reduced to a minimum spanning tree network :param dict params: dictionary of configuration parameters :param list m_ifgs: list of multilooked Ifg class objects (sequence must be multilooked versions of 'ifgs' arg) :return: None - interferogram phase data is updated and saved to disk """ # pylint: disable=too-many-locals, too-many-arguments offset = params[C.ORBFIT_OFFSET] degree = params[C.ORBITAL_FIT_DEGREE] preread_ifgs = params[C.PREREAD_IFGS] intercept = params[C.ORBFIT_INTERCEPT] scale = params[C.ORBFIT_SCALE] # all orbit corrections available? if isinstance(ifg_paths[0], str): if __check_and_apply_orberrors_found_on_disc(ifg_paths, params): log.warning("Reusing orbfit errors from previous run!!!") return # all corrections are available in numpy files already saved - return ifgs = [shared.Ifg(i) for i in ifg_paths] else: # alternate test paths # TODO: improve ifgs = ifg_paths src_ifgs = ifgs if m_ifgs is None else m_ifgs src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst if preread_ifgs: temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values() ids = first_second_ids(get_all_epochs(temp_ifgs)) else: ids = first_second_ids(get_all_epochs(ifgs)) nepochs = len(set(ids)) # call the actual inversion routine coefs = calc_network_orb_correction(src_ifgs, degree, scale, nepochs, intercept=intercept) # create full res DM to expand determined coefficients into full res # orbital correction (eg. expand coarser model to full size) if preread_ifgs: temp_ifg = Ifg(ifg_paths[0]) # ifgs here are paths temp_ifg.open() dm = get_design_matrix(temp_ifg, degree, intercept=intercept, scale=scale) temp_ifg.close() else: ifg = ifgs[0] dm = get_design_matrix(ifg, degree, intercept=intercept, scale=scale) for i in ifg_paths: # open if not Ifg instance if isinstance(i, str): # pragma: no cover # are paths i = Ifg(i) i.open(readonly=False) shared.nan_and_mm_convert(i, params) _remove_network_orb_error(coefs, dm, i, ids, offset, params)