def combine_data(self, filename):
"""
Combine data from multiple stations. First chronological station is the reference one.
"""
delete_stat = []
with open(filename, 'r') as fid:
# Loop over lines in file
for line in fid:
# Read the statnames
statnames = [name.lower().strip() for name in line.split(',')]
# Sort the stations by earliest finite data
starting_indices = []
for name in statnames:
data = self.statDict[name][self.components[0]]
starting_indices.append(np.isfinite(data).nonzero()[0][0])
statnames = [
statnames[ind] for ind in np.argsort(starting_indices)
]
ref_name = statnames[0]
# Loop over the components
for comp in self.components:
# Starting time representation is linear polynomial
rep = [['POLY', [1], [0.0]]]
# Get reference array to store everything
data = self.statDict[ref_name][comp]
wgt = self.statDict[ref_name]['w_' + comp]
for current_name in statnames[1:]:
# Get current array
current_data = self.statDict[current_name][comp]
current_wgt = self.statDict[current_name]['w_' + comp]
ind = np.isfinite(current_data).nonzero()[0]
# Store the data and weights
data[ind[0]:] = current_data[ind[0]:]
wgt[ind[0]:] = current_wgt[ind[0]:]
# Add heaviside
rep.append(['STEP', [self.trel[ind[0]]]])
# Remove ambiguities for any non-vertical components
if comp != 'up':
# Do least squares on finite data
ind = np.isfinite(data)
G = ts.Timefn(rep, self.trel)[0]
m = np.linalg.lstsq(G[ind, :], data[ind])[0]
# Remove only heaviside parts
data -= np.dot(G[:, 2:], m[2:])
# Remember the stations to delete
delete_stat.extend(statnames[1:])
# Delete redundant stations
for name in delete_stat:
del self.statDict[name]
# Some cleanup: remake the station generator
self.makeStatGen()
self.transferDictInfo()
return
def makeRepresentation(tdec, rank, splineOrder=3, secular=False, maxspl=2048):
"""
Constructs a highly overcomplete dictionary for approximately shift-invariant sparsity.
"""
# Check that the size of tdec is a power of two
N = tdec.size
assert np.log2(N) % int(np.log2(N)) < 1.0e-8, \
'Size of input data is not a power of two'
# Determine timescales to fill in G
N_levels = int(np.log2(N))
levels = 0
istart = 2
for j in range(N_levels - 2, -1, -1):
nspl = 2**(j + 2)
if nspl > maxspl:
istart += 1
continue
levels += 1
# Now construct G
G = np.zeros((N, N * levels))
cnt = 0
for j in range(N_levels - istart, -1, -1):
# Determine timescale of I-spline for current dyadic level
nspl = 2**(j + 2)
if nspl > maxspl:
cnt += 1
continue
tk = np.linspace(tdec.min(), tdec.max(), nspl)
dtk = abs(tk[1] - tk[0])
if rank == 0:
print('For', nspl, 'splines, tau is', 2 * dtk, 'years or',
2 * dtk * 365.0, 'days')
# Loop over time shifts corresponding to every observation epoch
for p in range(N):
hfn = ispline(splineOrder, dtk, tdec - tdec[p])
G[:, N * cnt + p] = hfn
cnt += 1
# Add secular components if requested
if secular:
rep = [['OFFSET', [0.0]], ['LINEAR', [0.0]]]
Gsec = np.asarray(ts.Timefn(rep, tdec - tdec[0])[0], order='C')
G = np.column_stack((Gsec, G))
return G
def _rep2matrix(self):
"""
Convert the string representation to a numpy array
"""
# Use Timefn to get matrix
self._matrix, mName, regF = ts.Timefn(self.rep, self.t)
self.regF = regF
# Modulate by a connectivity matrix (for InSAR)
if self.Jmat is not None:
self._matrix = np.dot(self.Jmat, self._matrix)
# Determine indices for non-regularized variables
self.noreg_ind = (regF < 0.9).nonzero()[0]
# And indices for regularized variables
self.reg_ind = (regF > 0.1).nonzero()[0]
return
def seasonalModulation(self, seasonalfile):
"""
Open an Insar stack containing seasonal signals for every station/pixel.
This will be used to create modulated seasonal matrix.
"""
# First build the modulation design matrix
for key in self.repKeys:
values = self.repDict[key]
repType = key.upper()
if 'modulated' in key:
rep = [['ISPLINE', values]]
seas_matrix = ts.Timefn(rep, self.t)[0]
break
# Load an Insar stack for the seasonal data
comm = comm or MPI.COMM_WORLD
data = Insar(name='seasonal', comm=comm)
def appendSeasonalDictionary(G, G_mod_ref, data):
"""
Prepends seasonal temporal dictionary to an existing temporal dictionary. The
seasonal dictionary is represented by modulating integrated B-splines for a template
of repeating B-splines.
"""
import tsinsar as ts
npbspline = data.npbspline
G_seas = ts.Timefn([['PBSPLINES', [3], [npbspline], 1.0]], data.trel)[0]
template = np.dot(G_seas, self.m_seas[jj, :])
# Normalize the template
mean_spline = np.mean(template)
fit_norm = template - mean_spline
template = fit_norm / (0.5 * (fit_norm.max() - fit_norm.min()))
G_mod = G_mod_ref.copy()
for nn in range(nsplmod):
G_mod[:, nn] *= template
G = np.column_stack((G_mod, self.G))
def preprocess(self, hdrformat='filt', dataFactor=1000.0):
"""
Preprocess the data to remove any known offsets as listed in the
Sopac header file. Only for Sopac data formats
"""
# Don't do anything if we're not using Sopac
if self.datformat != 'sopac':
return
if hdrformat == 'filt':
csopac = ts.sopac.sopac
elif hdrformat == 'trend':
csopac = sopac
# Loop through stations
print('Preprocessing for realz')
for stn in self.stns:
smodel = csopac(stn.fname)
components = [smodel.north, smodel.east, smodel.up]
data = [stn.north, stn.east, stn.up]
# Loop through components
for ii in range(len(data)):
comp = components[ii]
# Get representation for offset
frep = comp.offset
if len(frep) < 1:
continue
rep = []
amp = []
for crep in frep:
print(stn.fname, crep.rep, crep.amp)
rep.append(crep.rep)
amp.append(crep.amp)
# Construct design matrix
plt.plot(stn.tdec, data[ii], 'o')
G = np.asarray(ts.Timefn(rep, stn.tdec)[0], order='C')
amp = dataFactor * np.array(amp)
# Compute modeled displacement and remove from data
fit = np.dot(G, amp)
data[ii] -= fit
def distributem(self, reconstruct=True, transient=False, nsplmod=8):
"""
Distribute the solution coefficients into the data object station dictionary.
"""
if transient:
cutoff = self.cutoff
else:
cutoff = 0
# Make a seasonal amplitude modulator if necessary
if self.data.have_seasonal:
rep = [['ISPLINE', [3], [nsplmod]]]
G_mod_ref = ts.Timefn(rep, self.data.trel)[0]
if self.rank == 0:
ind = 0
for component in self.components:
for statname, stat in self.data.statGen:
m = self.m0[ind, :]
# Check if a seasonal representation has been specified
if self.data.have_seasonal:
G_seas = ts.Timefn(self.timeRep.rep, self.data.trel)[0]
template = np.dot(G_seas, self.m_seas0[ind, :])
# Normalize the template
mean_spline = np.mean(template)
fit_norm = template - mean_spline
template = fit_norm / (
0.5 * (fit_norm.max() - fit_norm.min()))
G_mod = G_mod_ref.copy()
for nn in range(nsplmod):
G_mod[:, nn] *= template
G = np.column_stack((G_mod, self.G))
else:
G = self.G
recon = np.dot(G[:, cutoff:], m[cutoff:])
signal_steady = np.dot(G[:, nsplmod:cutoff],
m[nsplmod:cutoff])
signal_seasonal = np.dot(G[:, :nsplmod], m[:nsplmod])
signal_remove = signal_steady + signal_seasonal
#recon = np.dot(G[:,:nsplmod], m[:nsplmod])
#signal_remove = np.dot(G[:,nsplmod:], m[nsplmod:])
try:
setattr(stat, 'm_' + component, self.m0[ind, :])
if reconstruct:
dat = getattr(stat, component)
setattr(stat, 'filt_' + component, recon)
setattr(stat, component, dat - signal_remove)
setattr(stat, 'steady_' + component, signal_remove)
except AttributeError:
stat['m_' + component] = self.m0[ind, :]
if reconstruct:
dat = np.array(stat[component])
stat['filt_' + component] = recon
stat[component] = dat - signal_remove
stat['steady_' + component] = signal_steady
stat['seasonal_' + component] = signal_seasonal
ind += 1
return
def invert(self,
maxiter=1,
weightingMethod='log',
wtype='median',
norm_tol=1.0e-4,
nsplmod=8):
# Choose my estimator for the weights
if 'mean' == wtype:
estimator = MPWeights.computeMean
elif 'median' == wtype:
estimator = MPWeights.computeMedian
else:
assert False, 'unsupported weight type. must be mean or median'
# Instantiate a solver
objSolver = self.solverClass(cutoff=self.cutoff,
maxiter=1,
eps=1.0e-2,
weightingMethod=weightingMethod)
# Make a seasonal amplitude modulator if necessary
if self.data.have_seasonal:
rep = [['ISPLINE', [3], [nsplmod]]]
G_mod_ref = ts.Timefn(rep, self.data.trel)[0]
self.nsplmod = nsplmod
else:
self.nsplmod = 0
# Iterate
nobs = self.nstat * self.ncomp
for ii in range(maxiter):
if self.rank == 0: print('At iteration', ii)
# Loop over my portion of the data
for jj in range(self.procN):
# Check if a seasonal representation has been specified
if self.data.have_seasonal:
G_seas = ts.Timefn(self.timeRep.rep, self.data.trel)[0]
template = np.dot(G_seas, self.m_seas[jj, :])
# Normalize the template
mean_spline = np.mean(template)
fit_norm = template - mean_spline
template = fit_norm / (0.5 *
(fit_norm.max() - fit_norm.min()))
G_mod = G_mod_ref.copy()
for nn in range(nsplmod):
G_mod[:, nn] *= template
G = np.column_stack((G_mod, self.G))
else:
G = self.G
# Get boolean indices of valid observations and weights
dat = self.datArr[jj, :].copy()
wgt = self.wgtArr[jj, :].copy()
ind = np.isfinite(dat) * np.isfinite(wgt)
# Subset the data and do the inversion
Gsub, dat, wgt = G[ind, :], dat[ind], wgt[ind]
# Get the penalties
λ = self.λ[jj, self.cutoff:]
# Perform estimation and store weights
self.m[jj, :], q = objSolver.invert(dmultl(wgt, Gsub),
wgt * dat, λ)
self.λ[jj, :] = q
# Wait for all workers to finish
self.comm.Barrier()
# Master gathers the results for this iteration
self.comm.Gatherv(
self.m, [self.m0, (self.sendcnts, None), self.par_rowtype],
root=0)
self.comm.Gatherv(
self.λ, [self.λ0, (self.sendcnts, None), self.par_rowtype],
root=0)
# Master computes the new weights
if self.rank == 0:
print(' - updating weights...')
exitFlag = self._updateWeights(estimator, norm_tol)
else:
exitFlag = None
# Scatter the updated penalties
self.comm.Scatterv(
[self.λ0, (self.sendcnts, None), self.par_rowtype],
self.λ,
root=0)
# Broadcast the exit flag
exitFlag = self.comm.bcast(exitFlag, root=0)
if exitFlag:
break
self.par_rowtype.Free()
return
fout.create_dataset('dates', data=da)
fout.create_dataset('tims', data=ti)
fout.create_dataset('masterind', data=masterind)
# Read data
igram = fdat['figram']
Ny = igram[:, stay:endy:stey, stax:endx:stex].shape[1]
Nx = igram[:, stay:endy:stey, stax:endx:stex].shape[2]
igramsub = fout.create_dataset('Data', (Nifg, Ny, Nx), 'f')
igramsub = igram[:, stay:endy:stey, stax:endx:stex]
tims = fdat['tims']
print 'Process %d: Build time function' % (rank)
rep = [['SBAS', masterind]]
H, mName, regF = ts.Timefn(rep, tims)
print 'Process %d: Build Generic G matrix' % (rank)
indJ = np.arange(Nsar)
indJ = np.delete(indJ, masterind)
indH = np.arange(H.shape[1])
indH = np.delete(indH, masterind)
Jmat = Jmat[:, indJ]
H = H[indJ, :]
H = H[:, indH]
nm = H.shape[1]
fout.create_dataset('Hmat', data=H)
Gg = np.dot(Jmat, H)
[xx, yy] = np.where(np.abs(Gg) < 1e-20)
Gg[xx, yy] = 0.0
daylist = ts.datestr(dates)
########Master time and index information
masterdate = (ppars.params.proc.masterdate)
if masterdate is None:
masterind = 0
else:
masterind = daylist.index(masterdate)
tmaster = tims[masterind]
#####Load the dictionary from a user defined dictionary.
rep = user.timedict()
regu = (ppars.params.proc.regularize)
H, mName, regF = ts.Timefn(rep, tims) #Greens function matrix
######Looking up master index.
ind = np.arange(Nsar)
ind = np.delete(ind, masterind)
Jmat[:, masterind] = 0.0
nm = H.shape[1]
#######Setting up regularization
nReg = np.int(regF.max())
if (nReg == 0) & (regu):
logging.info('Nothing to Regularize')
L = []
def extended_spinvert(self,
tdec,
repDict,
penalty,
cutoffDict,
maxiter=4,
outlierThresh=1.0e6):
"""
Performs sparse inversion of model coefficients on each component. Each
station will have its own time representation and its own cutoff.
"""
# Loop over the stations
mDicts = {}
for statname, stat in self.statGen:
# Construct a G matrix
Gref = np.asarray(ts.Timefn(repDict[statname], tdec - tdec[0])[0],
order='C')
ndat, Npar = Gref.shape
refCutoff = cutoffDict[statname]
# Loop over the components
mDict = {}
for comp, w_comp in [('east', 'w_e'), ('north', 'w_n'),
('up', 'w_u')]:
#if comp in ['east', 'north']:
# G = Gref[:,4:]
# cutoff = refCutoff - 4
#else:
# G = Gref
# cutoff = refCutoff
cutoff = refCutoff
G = Gref
# Get finite data
dat = (getattr(stat, comp)).copy()
ind = np.isfinite(dat)
dat = dat[ind]
wgt = getattr(stat, w_comp)[ind]
# Instantiate a solver
solver = sp.BaseOpt(cutoff=cutoff,
maxiter=maxiter,
weightingMethod='log')
# Perform estimation
m = solver.invert(dmultl(wgt, G[ind, :]), wgt * dat,
penalty)[0]
# Do one pass to remove outliers
fit = np.dot(G, m)
raw_dat = getattr(stat, comp)
misfit = np.abs(raw_dat - fit)
ind = misfit > outlierThresh
if ind.nonzero()[0].size > 1:
print('Doing another pass to remove outliers')
raw_dat[ind] = np.nan
finiteInd = np.isfinite(raw_dat)
dat = raw_dat[finiteInd]
wgt = getattr(stat, w_comp)[finiteInd]
m = solver.invert(dmultl(wgt, G[finiteInd, :]), wgt * dat,
penalty)[0]
#if comp in ['east', 'north']:
# m = np.hstack((np.zeros((4,)), m))
mDict[comp] = m
mDicts[statname] = (mDict, G)
return mDicts