def gen_BandMat_simple(size):
"""Generates a random BandMat."""
l = random.choice([0, 1, randint(0, 10)])
u = random.choice([0, 1, randint(0, 10)])
data = randn(l + u + 1, size)
transposed = rand_bool()
return bm.BandMat(l, u, data, transposed=transposed)
def prepare_mats(refl, alpha=1e3):
"""
Does some prep to produce matrices for the TDMA
"""
y = refl
Iobs = y > 0
nT, ys, xs = y.shape
# Create mats
AC, BC, CC, DC = [ np.zeros([nT-1, ys, xs], dtype=np.float32),
np.zeros([nT, ys, xs], dtype=np.float32),
np.zeros([nT-1, ys, xs], dtype=np.float32),
np.zeros([nT, ys, xs], dtype=np.float32)]
# Create Diagonals matrices
# for ease just use bandmat for this for now
# Form diagonals
I = np.eye(nT) #np.diag (np.ones(nx))
D = (I - np.roll(I, -1)).T
#D = D.T.dot(D)
D1A = np.zeros((nT, 2))
D1A[1:, 0] = np.diag(D, 1)
D1A[:, 1] = np.diag(D, 0)
# convert to banded matrices
D = bm.BandMat(0, 1, D1A.T, transposed=False)
D2 = bm.dot_mm(D.T, D)
a = alpha * D2.data[2][:-1]
b = alpha * D2.data[1]
c = alpha * D2.data[0][1:]
# add these
AC[:]=a[:, None, None]
BC[:]=b[:, None, None]+ Iobs
CC[:]=c[:, None, None]
DC[:]=y
return AC, BC, CC, DC
def smooth(a, Phat, N, lambda_1, lambda_2):
"""
a: (N,) number of measurements at that timestep
Phat: (N, 3) sum of measurements at that timestep
N: num time steps
lambda_1, lambda_2: regularization parameters
solves the optimization problem (over P \in R^{Tx3}):
minimize ||diag(a)*P-Phat||^2 + lambda_1/N*||D_2*P||^2 + lambda_2/N*||D_3*P||^2
returns:
- P: (N, 3) matrix with full trajectory
"""
# A in Banded Matrix form
A = bm.diag(1. * a)
# D_2 and D_3 in Banded Matrix form transposed
D_2_bm_T = bm.BandMat(
1, 1,
np.hstack([
np.zeros((3, 1)),
np.repeat([[1.], [-2.], [1.]], N - 2, axis=1),
np.zeros((3, 1))
]))
D_3_bm_T = bm.BandMat(
2, 2,
np.hstack([
np.zeros((5, 2)),
np.repeat([[-1.], [2.], [0.], [-2.], [1.]], N - 4, axis=1),
np.zeros((5, 2))
]))
# XP=B normal equations
X = bm.dot_mm(A, A) + lambda_1 / N * bm.dot_mm(
D_2_bm_T, D_2_bm_T.T) + lambda_2 / N * bm.dot_mm(D_3_bm_T, D_3_bm_T.T)
l_and_u = (X.l, X.u) # lower and upper band bounds
B = np.hstack([
np.expand_dims(bm.dot_mv(A, Phat[:, 0]), -1),
np.expand_dims(bm.dot_mv(A, Phat[:, 1]), -1),
np.expand_dims(bm.dot_mv(A, Phat[:, 2]), -1)
])
# solve normal equations
P = solve_banded(l_and_u, X.data, B)
return P
def test__cholesky_banded(self, its=100):
for it in range(its):
size = random.choice([0, 1, randint(0, 10), randint(0, 100)])
if rand_bool():
mat_bm = gen_pos_def_BandMat(size, transposed=False)
else:
mat_bm = gen_symmetric_BandMat(size, transposed=False)
# make it a bit more likely to be pos def
bm.diag(mat_bm)[:] = np.abs(bm.diag(mat_bm)) + 0.1
depth = mat_bm.l
lower = rand_bool()
if lower:
mat_half_data = mat_bm.data[depth:]
else:
mat_half_data = mat_bm.data[:(depth + 1)]
overwrite = rand_bool()
mat_half_data_arg = mat_half_data.copy()
try:
chol_data = bla._cholesky_banded(mat_half_data_arg,
overwrite_ab=overwrite,
lower=lower)
except la.LinAlgError as e:
# First part of the message is e.g. "2-th leading minor".
msgRe = (r'^' + re.escape(str(e)[:15]) +
r'.*not positive definite$')
with self.assertRaisesRegexp(la.LinAlgError, msgRe):
sla.cholesky(mat_bm.full(), lower=lower)
else:
assert np.shape(chol_data) == (depth + 1, size)
if lower:
chol_bm = bm.BandMat(depth, 0, chol_data)
mat_bm_again = bm.dot_mm(chol_bm, chol_bm.T)
else:
chol_bm = bm.BandMat(0, depth, chol_data)
mat_bm_again = bm.dot_mm(chol_bm.T, chol_bm)
assert_allclose(mat_bm_again.full(), mat_bm.full())
if size > 0:
self.assertEqual(
np.may_share_memory(chol_data, mat_half_data_arg),
overwrite)
if not overwrite:
assert np.all(mat_half_data_arg == mat_half_data)
def gen_BandMat(size, l=None, u=None, transposed=None):
"""Generates a random BandMat."""
if l is None:
l = random.choice([0, 1, randint(0, 10)])
if u is None:
u = random.choice([0, 1, randint(0, 10)])
data = randn(l + u + 1, size)
if transposed is None:
transposed = rand_bool()
return bm.BandMat(l, u, data, transposed=transposed)
def make_D_matrix(nT):
"""
a function to make a sparse D matrix
"""
I = np.eye(nT) #np.diag (np.ones(nx))
D = (I - np.roll(I, -1)).T
#D = D.T.dot(D)
D1A = np.zeros((nT, 2))
D1A[1:, 0] = np.diag(D, 1)
D1A[:, 1] = np.diag(D, 0)
# convert to banded matrices
D = bm.BandMat(0, 1, D1A.T, transposed=False)
return D
#!/usr/bin/python
"""A simple example of how the bandmat package may be used."""
# Copyright 2013, 2014, 2015, 2016, 2017, 2018 Matt Shannon
# This file is part of bandmat.
# See `License` for details of license and warranty.
import numpy as np
import bandmat as bm
a_bm = bm.BandMat(
1, 1,
np.array([
[0.0, 0.2, 0.3, 0.4, 0.5],
[1.0, 0.9, 1.1, 0.8, 1.3],
[0.3, 0.1, 0.5, 0.6, 0.0],
]))
b_bm = bm.BandMat(
1, 0, np.array([
[1.0, 0.9, 1.1, 0.8, 1.3],
[0.3, 0.1, 0.5, 0.6, 0.0],
]))
c_bm = bm.dot_mm(a_bm.T, b_bm)
d_bm = a_bm + b_bm
a_full = a_bm.full()
def solve_band(refl, solve_edge=True, W=None, alpha=1000, unc=False, drop=True):
"""
test run function
"""
Iobs = (refl>0)
nObs = (Iobs).sum(axis=0)
# scale alpha
alpha = 1
alpha = alpha * np.nanmean(nObs)
alpha = np.clip(alpha, 20, 250)
if solve_edge:
T = .6 * alpha
nT, ys, xs = refl.shape
# make D matrix
I = np.eye(nT) #np.diag (np.ones(nx))
D = (I - np.roll(I, -1)).T
#D = D.T.dot(D)
D1A = np.zeros((nT, 2))
D1A[1:, 0] = np.diag(D, 1)
D1A[:, 1] = np.diag(D, 0)
# convert to banded matrices
_D = bm.BandMat(0, 1, D1A.T, transposed=False)
# create mats
A, B, C, D = prepare_mats(refl, alpha=alpha)
# Initial run with no smoothing
X = TDMA_MAT(A, B, C, D)
W0 = np.ones_like(X)*100
WW = np.ones_like(X)
X0 = np.zeros_like(D)
CONV = np.zeros((ys, xs)).astype(np.bool)
Nits = np.zeros((ys, xs), dtype=np.int)
for i in range(5):
# calculate weights
order_n = 1
#dx = np.diff(X, axis=0, n=order_n)* alpha**2
dx = np.gradient(X, axis=0) * alpha**2
# enforce direction constraint
if drop != None:
if drop:
dx[dx>0]=0
else:
dx[dx<=0]=0
# eg frst 30 days and last 30 days
_w = np.exp(-dx**2 / T)
"""
Scaling from CERC
"""
y,x=30, 57
wScale = nObs / np.sum(_w, axis=0)
_w = _w * wScale
_w = np.clip(_w, 0., 1)
np.place(_w, np.isnan(_w), 1)
WW[:]=_w
"""
Update the regularisation matrix
with the weights
-- This involves a dot product
with a diag eg D.T W D
so i don't how to do this atm
This is the slowest so if this can be done fast much better
"""
for y in range(ys):
for x in range(xs):
if not CONV[y,x]:
# no convergence keep going
n = alpha * bm.dot_mm(_D.T, _D, diag=WW[:, y, x].astype(np.float))
a = n.data[2][:-1]
b = n.data[1] + Iobs[:, y, x]
c = n.data[0][1:]
# update matrices
A[:, y, x]=a
B[:, y, x]=b
C[:, y, x]=c
Nits[y,x]+=1
"""
Run again with new weights
"""
X = TDMA_MAT(A, B, C, D)
"""
assess convergence
-eg are the weights still changing
"""
cow = np.sum(np.abs((WW - W0) / W0), axis=0)+ 1e-6
CONV[cow<1e-2]=True
MINITER=3
if i< MINITER:
CONV[:]=False
CONV[WW.min(axis=0)<0.1]=True
W0 = WW
X0 = X
"""
Want to check steps and re-inforce real steps
"""
W, keptEdges, Sch = refine_edges(WW, X, refl)
"""
Resolve with refined edges
"""
for y in range(ys):
for x in range(xs):
n = alpha * bm.dot_mm(_D.T, _D, diag=W[:, y, x].astype(np.float))
a = n.data[2][:-1]
b = n.data[1] + Iobs[:, y, x]
c = n.data[0][1:]
# update matrices
A[:, y, x]=a
B[:, y, x]=b
C[:, y, x]=c
Nits[y,x]+=1
"""
Run again with the refined weights
"""
#y,x=43, 62
if unc:
X = TDMA_MAT(A, B, C, D)
DD = np.copy(D).astype(np.float32)
Inv = np.zeros_like(X)
# only need the middle month tbh
for t in range(20, nT-20):
DD[:]=0
DD[t]=1
Inv[t]=TDMA_MAT(A, B, C, DD)[t]
return X, Inv, W, CONV, Nits, Sch
else:
X = TDMA_MAT(A, B, C, D)
return X, W, CONV, Nits, Sch
else:
"""
Edge has been provided so use that
"""
Iobs = (refl>0)
nObs = (Iobs).sum(axis=0)
nT, ys, xs = refl.shape
# make D matrix
I = np.eye(nT) #np.diag (np.ones(nx))
D = (I - np.roll(I, -1)).T
#D = D.T.dot(D)
D1A = np.zeros((nT, 2))
D1A[1:, 0] = np.diag(D, 1)
D1A[:, 1] = np.diag(D, 0)
# convert to banded matrices
_D = bm.BandMat(0, 1, D1A.T, transposed=False)
# create mats
A, B, C, D = prepare_mats(refl, alpha=alpha)
# add the weights matrix
for y in range(ys):
for x in range(xs):
n = alpha * bm.dot_mm(_D.T, _D, diag=W[:, y, x].astype(np.float))
a = n.data[2][:-1]
b = n.data[1] + Iobs[:, y, x]
c = n.data[0][1:]
# update matrices
A[:, y, x]=a
B[:, y, x]=b
C[:, y, x]=c
"""
Run again with the refined weights
"""
X = TDMA_MAT(A, B, C, D)
return X, W