def test_master_slave_ids(self):
d0 = date(2006, 6, 19)
d1 = date(2006, 8, 28)
d2 = date(2006, 10, 2)
d3 = date(2006, 11, 6)
exp = {d0: 0, d1: 1, d2: 2, d3: 3}
# test unordered and with duplicates
self.assertEqual(exp, master_slave_ids([d3, d0, d2, d1]))
self.assertEqual(exp,
master_slave_ids([d3, d0, d2, d1, d3, d0]))
def get_date_ids(ifgs):
'''
Returns unique master/slave date IDs from the given Ifgs.
'''
dates = []
for ifg in ifgs:
dates += [ifg.master, ifg.slave]
return algorithm.master_slave_ids(dates)
def get_network_design_matrix(ifgs, degree, offset):
# pylint: disable=too-many-locals
"""
Returns larger-format design matrix for network error correction. The
network design matrix includes rows which relate to those of NaN cells.
:param list ifgs: List of Ifg class objects
:param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC)
:param bool offset: True to include offset cols, otherwise False.
:return: netdm: network design matrix
:rtype: ndarray
"""
if degree not in [PLANAR, QUADRATIC, PART_CUBIC]:
raise OrbitalError("Invalid degree argument")
nifgs = len(ifgs)
if nifgs < 1:
# can feasibly do correction on a single Ifg/2 epochs
raise OrbitalError("Invalid number of Ifgs: %s" % nifgs)
# init sparse network design matrix
nepochs = len(set(get_all_epochs(ifgs)))
# no offsets: they are made separately below
ncoef = _get_num_params(degree)
shape = [ifgs[0].num_cells * nifgs, ncoef * nepochs]
if offset:
shape[1] += nifgs # add extra block for offset cols
netdm = zeros(shape, dtype=float32)
# calc location for individual design matrices
dates = [ifg.master for ifg in ifgs] + [ifg.slave for ifg in ifgs]
ids = master_slave_ids(dates)
offset_col = nepochs * ncoef # base offset for the offset cols
tmpdm = get_design_matrix(ifgs[0], degree, offset=False)
# iteratively build up sparse matrix
for i, ifg in enumerate(ifgs):
rs = i * ifg.num_cells # starting row
m = ids[ifg.master] * ncoef # start col for master
s = ids[ifg.slave] * ncoef # start col for slave
netdm[rs:rs + ifg.num_cells, m:m + ncoef] = -tmpdm
netdm[rs:rs + ifg.num_cells, s:s + ncoef] = tmpdm
# offsets are diagonal cols across the extra array block created above
if offset:
netdm[rs:rs + ifg.num_cells,
offset_col + i] = 1 # init offset cols
return netdm
def get_vcmt(ifgs, maxvar):
"""
Assembles a temporal variance/covariance matrix using the method
described by Biggs et al., Geophys. J. Int, 2007. Matrix elements are
evaluated according to sig_i * sig_j * C_ij where i and j are two
interferograms and C is a matrix of coefficients:
C = 1 if the master and slave epochs of i and j are equal
C = 0.5 if have i and j share either a common master or slave epoch
C = -0.5 if the master of i or j equals the slave of the other
C = 0 otherwise
:param list ifgs: A list of pyrate.shared.Ifg class objects.
:param ndarray maxvar: numpy array of maximum variance values for the
interferograms.
:return: vcm_t: temporal variance-covariance matrix
:rtype: ndarray
"""
# pylint: disable=too-many-locals
# c=0.5 for common master or slave; c=-0.5 if master
# of one matches slave of another
if isinstance(ifgs, dict):
from collections import OrderedDict
ifgs = {k: v for k, v in ifgs.items() if isinstance(v, PrereadIfg)}
ifgs = OrderedDict(sorted(ifgs.items()))
# pylint: disable=redefined-variable-type
ifgs = ifgs.values()
nifgs = len(ifgs)
vcm_pat = zeros((nifgs, nifgs))
dates = [ifg.master for ifg in ifgs] + [ifg.slave for ifg in ifgs]
ids = master_slave_ids(dates)
for i, ifg in enumerate(ifgs):
mas1, slv1 = ids[ifg.master], ids[ifg.slave]
for j, ifg2 in enumerate(ifgs):
mas2, slv2 = ids[ifg2.master], ids[ifg2.slave]
if mas1 == mas2 or slv1 == slv2:
vcm_pat[i, j] = 0.5
if mas1 == slv2 or slv1 == mas2:
vcm_pat[i, j] = -0.5
if mas1 == mas2 and slv1 == slv2:
vcm_pat[i, j] = 1.0 # diagonal elements
# make covariance matrix in time domain
std = sqrt(maxvar).reshape((nifgs, 1))
vcm_t = std * std.transpose()
return vcm_t * vcm_pat
def _time_series_setup(ifgs, mst, params):
"""
Convenience function for setting up time series computation parameters
"""
if len(ifgs) < 1:
msg = 'Time series requires 2+ interferograms'
raise TimeSeriesError(msg)
# if mst is not a single tree then do interpolation
interp = 0 if mst_module.mst_from_ifgs(ifgs)[1] else 1
# Parallel Processing parameters
parallel = params[cf.PARALLEL]
# Time Series parameters
tsmethod = params[cf.TIME_SERIES_METHOD]
pthresh, smfactor, smorder = _validate_params(params, tsmethod)
epochlist = get_epochs(ifgs)[0]
nrows = ifgs[0].nrows
ncols = ifgs[0].ncols
nifgs = len(ifgs)
span = diff(epochlist.spans)
nepoch = len(epochlist.dates) # epoch number
nvelpar = nepoch - 1 # velocity parameters number
# nlap = nvelpar - smorder # Laplacian observations number
mast_slave_ids = master_slave_ids(epochlist.dates)
imaster = [mast_slave_ids[ifg.master] for ifg in ifgs]
islave = [mast_slave_ids[ifg.slave] for ifg in ifgs]
imaster = min(imaster, islave)
islave = max(imaster, islave)
b0_mat = zeros((nifgs, nvelpar))
for i in range(nifgs):
b0_mat[i, imaster[i]:islave[i]] = span[imaster[i]:islave[i]]
# change the sign if slave is earlier than master
isign = where(imaster > islave)
b0_mat[isign[0], :] = -b0_mat[isign[0], :]
tsvel_matrix = np.empty(shape=(nrows, ncols, nvelpar),
dtype=float32)
ifg_data = np.zeros((nifgs, nrows, ncols), dtype=float32)
for ifg_num in range(nifgs):
ifg_data[ifg_num] = ifgs[ifg_num].phase_data
if mst is None:
mst = ~isnan(ifg_data)
return b0_mat, interp, pthresh, smfactor, smorder, tsmethod, ifg_data, \
mst, ncols, nrows, nvelpar, parallel, span, tsvel_matrix
def get_vcmt(ifgs, maxvar):
"""
Returns the temporal variance/covariance matrix.
:param ifgs: xxxx
:param maxvar: xxxx
:return xxxx
"""
# pylint: disable=too-many-locals
# c=0.5 for common master or slave; c=-0.5 if master
# of one matches slave of another
if isinstance(ifgs, dict):
from collections import OrderedDict
ifgs = {k: v for k, v in ifgs.items() if isinstance(v, PrereadIfg)}
ifgs = OrderedDict(sorted(ifgs.items()))
# pylint: disable=redefined-variable-type
ifgs = ifgs.values()
nifgs = len(ifgs)
vcm_pat = zeros((nifgs, nifgs))
dates = [ifg.master for ifg in ifgs] + [ifg.slave for ifg in ifgs]
ids = master_slave_ids(dates)
for i, ifg in enumerate(ifgs):
mas1, slv1 = ids[ifg.master], ids[ifg.slave]
for j, ifg2 in enumerate(ifgs):
mas2, slv2 = ids[ifg2.master], ids[ifg2.slave]
if mas1 == mas2 or slv1 == slv2:
vcm_pat[i, j] = 0.5
if mas1 == slv2 or slv1 == mas2:
vcm_pat[i, j] = -0.5
if mas1 == mas2 and slv1 == slv2:
vcm_pat[i, j] = 1.0 # handle testing ifg against itself
# make covariance matrix in time domain
std = sqrt(maxvar).reshape((nifgs, 1))
vcm_t = std * std.transpose()
return vcm_t * vcm_pat
def network_orbital_correction(ifgs,
degree,
offset,
params,
m_ifgs=None,
preread_ifgs=None):
"""
This algorithm implements a network inversion to determine orbital
corrections for a set of interferograms forming a connected network.
Warning: This will write orbital error corrected phase_data to the ifgs.
:param list ifgs: List of Ifg class objects reduced to a minimum spanning
tree network
:param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC)
:param bool offset: True to calculate the model using offsets
:param dict params: dictionary of configuration parameters
:param list m_ifgs: list of multilooked Ifg class objects
(sequence must be multilooked versions of 'ifgs' arg)
:param dict preread_ifgs: Dictionary containing information specifically
for MPI jobs (optional)
:return: None - interferogram phase data is updated and saved to disk
"""
# pylint: disable=too-many-locals, too-many-arguments
src_ifgs = ifgs if m_ifgs is None else m_ifgs
src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst
vphase = vstack([i.phase_data.reshape((i.num_cells, 1)) for i in src_ifgs])
vphase = squeeze(vphase)
B = get_network_design_matrix(src_ifgs, degree, offset)
# filter NaNs out before getting model
B = B[~isnan(vphase)]
orbparams = dot(pinv(B, 1e-6), vphase[~isnan(vphase)])
ncoef = _get_num_params(degree)
if preread_ifgs:
temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values()
ids = master_slave_ids(get_all_epochs(temp_ifgs))
else:
ids = master_slave_ids(get_all_epochs(ifgs))
coefs = [
orbparams[i:i + ncoef] for i in range(0,
len(set(ids)) * ncoef, ncoef)
]
# create full res DM to expand determined coefficients into full res
# orbital correction (eg. expand coarser model to full size)
if preread_ifgs:
temp_ifg = Ifg(ifgs[0]) # ifgs here are paths
temp_ifg.open()
dm = get_design_matrix(temp_ifg, degree, offset=False)
temp_ifg.close()
else:
dm = get_design_matrix(ifgs[0], degree, offset=False)
for i in ifgs:
# open if not Ifg instance
if isinstance(i, str): # pragma: no cover
# are paths
i = Ifg(i)
i.open(readonly=False)
shared.nan_and_mm_convert(i, params)
_remove_network_orb_error(coefs, dm, i, ids, offset)
def network_correction(ifgs,
degree,
offset,
params,
m_ifgs=None,
preread_ifgs=None):
"""
Calculates orbital correction model, removing this from the interferograms.
.. warn:: This will write orbital error corrected phase_data in the interferograms.
:param ifgs: Interferograms reduced to a minimum tree from prior
MST calculations
:param degree: PLANAR, QUADRATIC or PART_CUBIC
:param offset: True to calculate the model using offsets
:param params: Parameter dictionary
:param m_ifgs: Multi-looked orbfit interferograms (sequence must be mlooked
versions of 'ifgs' arg)
:param preread_ifgs: Parameters dict corresponding to config file
:return xxxx
"""
# pylint: disable=too-many-locals, too-many-arguments
src_ifgs = ifgs if m_ifgs is None else m_ifgs
src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst
vphase = vstack([i.phase_data.reshape((i.num_cells, 1)) for i in src_ifgs])
vphase = squeeze(vphase)
B = get_network_design_matrix(src_ifgs, degree, offset)
# filter NaNs out before getting model
B = B[~isnan(vphase)]
orbparams = dot(pinv(B, 1e-6), vphase[~isnan(vphase)])
ncoef = _get_num_params(degree)
if preread_ifgs:
temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values()
ids = master_slave_ids(get_all_epochs(temp_ifgs))
else:
ids = master_slave_ids(get_all_epochs(ifgs))
coefs = [
orbparams[i:i + ncoef] for i in range(0,
len(set(ids)) * ncoef, ncoef)
]
# create full res DM to expand determined coefficients into full res
# orbital correction (eg. expand coarser model to full size)
if preread_ifgs:
temp_ifg = Ifg(ifgs[0]) # ifgs here are paths
temp_ifg.open()
dm = get_design_matrix(temp_ifg, degree, offset=False)
temp_ifg.close()
else:
dm = get_design_matrix(ifgs[0], degree, offset=False)
# TODO: remove this import from tests suite
from tests.common import MockIfg
for i in ifgs:
# open if not Ifg instance
if not (isinstance(i, Ifg)
or isinstance(i, MockIfg)): # pragma: no cover
# are paths
i = Ifg(i)
i.open(readonly=False)
shared.nan_and_mm_convert(i, params)
_remove_networkx_error(coefs, dm, i, ids, offset)
def time_series_setup(ifgs, mst, params):
"""
Convenience function setting up time series computation parameters.
:param ifgs: xxxx
:param mst: xxxx
:param params: xxxx
:return xxxx
"""
if len(ifgs) < 1:
msg = 'Time series requires 2+ interferograms'
raise TimeSeriesError(msg)
# if mst is not a single tree then do interpolation
interp = 0 if mst_module.mst_from_ifgs(ifgs)[1] else 1
# Parallel Processing parameters
parallel = params[cf.PARALLEL]
# Time Series parameters
tsmethod = params[cf.TIME_SERIES_METHOD]
if tsmethod == 1 and params[cf.TIME_SERIES_SM_ORDER] is None:
missing_option_error(cf.TIME_SERIES_SM_ORDER)
else:
smorder = params[cf.TIME_SERIES_SM_ORDER]
if tsmethod == 1 and params[cf.TIME_SERIES_SM_FACTOR] is None:
missing_option_error(cf.TIME_SERIES_SM_FACTOR)
else:
smfactor = np.power(10, params[cf.TIME_SERIES_SM_FACTOR])
if params[cf.TIME_SERIES_PTHRESH] is None:
missing_option_error(cf.TIME_SERIES_PTHRESH)
else:
pthresh = params[cf.TIME_SERIES_PTHRESH]
if pthresh < 0.0 or pthresh > 1000:
raise ValueError(
"minimum number of coherent observations for a pixel"
" TIME_SERIES_PTHRESH setting must be >= 0.0 and <= 1000")
epochlist = get_epochs(ifgs)[0]
nrows = ifgs[0].nrows
ncols = ifgs[0].ncols
nifgs = len(ifgs)
span = diff(epochlist.spans)
nepoch = len(epochlist.dates) # epoch number
nvelpar = nepoch - 1 # velocity parameters number
# nlap = nvelpar - smorder # Laplacian observations number
mast_slave_ids = master_slave_ids(epochlist.dates)
imaster = [mast_slave_ids[ifg.master] for ifg in ifgs]
islave = [mast_slave_ids[ifg.slave] for ifg in ifgs]
imaster = min(imaster, islave)
islave = max(imaster, islave)
b0_mat = zeros((nifgs, nvelpar))
for i in range(nifgs):
b0_mat[i, imaster[i]:islave[i]] = span[imaster[i]:islave[i]]
# change the sign if slave is earlier than master
isign = where(imaster > islave)
b0_mat[isign[0], :] = -b0_mat[isign[0], :]
tsvel_matrix = np.empty(shape=(nrows, ncols, nvelpar), dtype=float32)
ifg_data = np.zeros((nifgs, nrows, ncols), dtype=float32)
for ifg_num in range(nifgs):
ifg_data[ifg_num] = ifgs[ifg_num].phase_data
if mst is None:
mst = ~isnan(ifg_data)
return b0_mat, interp, pthresh, smfactor, smorder, tsmethod, ifg_data, \
mst, ncols, nrows, nvelpar, parallel, span, tsvel_matrix