def test_band_of_inverse_from_chol(self, its=50):
for it in range(its):
size = random.choice([0, 1, randint(0, 10), randint(0, 100)])
chol_bm = gen_chol_factor_BandMat(size)
depth = chol_bm.l + chol_bm.u
band_of_inv_bm = bla.band_of_inverse_from_chol(chol_bm)
assert not np.may_share_memory(band_of_inv_bm.data, chol_bm.data)
mat_bm = (bm.dot_mm(chol_bm, chol_bm.T)
if chol_bm.u == 0 else bm.dot_mm(chol_bm.T, chol_bm))
band_of_inv_full_good = fl.band_ec(
depth, depth,
np.eye(0, 0) if size == 0 else la.inv(mat_bm.full()))
assert_allclose(band_of_inv_bm.full(), band_of_inv_full_good)
def test_band_of_inverse_from_chol(self, its=50):
for it in range(its):
size = random.choice([0, 1, randint(0, 10), randint(0, 100)])
chol_bm = gen_chol_factor_BandMat(size)
depth = chol_bm.l + chol_bm.u
band_of_inv_bm = bla.band_of_inverse_from_chol(chol_bm)
assert not np.may_share_memory(band_of_inv_bm.data, chol_bm.data)
mat_bm = (bm.dot_mm(chol_bm, chol_bm.T) if chol_bm.u == 0
else bm.dot_mm(chol_bm.T, chol_bm))
band_of_inv_full_good = fl.band_ec(
depth, depth,
np.eye(0, 0) if size == 0 else la.inv(mat_bm.full())
)
assert_allclose(band_of_inv_bm.full(), band_of_inv_full_good)
def prepare_mats(refl, alpha=1e3):
"""
Does some prep to produce matrices for the TDMA
"""
y = refl
Iobs = y > 0
nT, ys, xs = y.shape
# Create mats
AC, BC, CC, DC = [ np.zeros([nT-1, ys, xs], dtype=np.float32),
np.zeros([nT, ys, xs], dtype=np.float32),
np.zeros([nT-1, ys, xs], dtype=np.float32),
np.zeros([nT, ys, xs], dtype=np.float32)]
# Create Diagonals matrices
# for ease just use bandmat for this for now
# Form diagonals
I = np.eye(nT) #np.diag (np.ones(nx))
D = (I - np.roll(I, -1)).T
#D = D.T.dot(D)
D1A = np.zeros((nT, 2))
D1A[1:, 0] = np.diag(D, 1)
D1A[:, 1] = np.diag(D, 0)
# convert to banded matrices
D = bm.BandMat(0, 1, D1A.T, transposed=False)
D2 = bm.dot_mm(D.T, D)
a = alpha * D2.data[2][:-1]
b = alpha * D2.data[1]
c = alpha * D2.data[0][1:]
# add these
AC[:]=a[:, None, None]
BC[:]=b[:, None, None]+ Iobs
CC[:]=c[:, None, None]
DC[:]=y
return AC, BC, CC, DC
def test__cholesky_banded(self, its=100):
for it in range(its):
size = random.choice([0, 1, randint(0, 10), randint(0, 100)])
if rand_bool():
mat_bm = gen_pos_def_BandMat(size, transposed=False)
else:
mat_bm = gen_symmetric_BandMat(size, transposed=False)
# make it a bit more likely to be pos def
bm.diag(mat_bm)[:] = np.abs(bm.diag(mat_bm)) + 0.1
depth = mat_bm.l
lower = rand_bool()
if lower:
mat_half_data = mat_bm.data[depth:]
else:
mat_half_data = mat_bm.data[:(depth + 1)]
overwrite = rand_bool()
mat_half_data_arg = mat_half_data.copy()
try:
chol_data = bla._cholesky_banded(
mat_half_data_arg, overwrite_ab=overwrite, lower=lower
)
except la.LinAlgError as e:
# First part of the message is e.g. "2-th leading minor".
msgRe = (r'^' + re.escape(str(e)[:15]) +
r'.*not positive definite$')
with self.assertRaisesRegexp(la.LinAlgError, msgRe):
sla.cholesky(mat_bm.full(), lower=lower)
else:
assert np.shape(chol_data) == (depth + 1, size)
if lower:
chol_bm = bm.BandMat(depth, 0, chol_data)
mat_bm_again = bm.dot_mm(chol_bm, chol_bm.T)
else:
chol_bm = bm.BandMat(0, depth, chol_data)
mat_bm_again = bm.dot_mm(chol_bm.T, chol_bm)
assert_allclose(mat_bm_again.full(), mat_bm.full())
if size > 0:
self.assertEqual(
np.may_share_memory(chol_data, mat_half_data_arg),
overwrite
)
if not overwrite:
assert np.all(mat_half_data_arg == mat_half_data)
def smooth(a, Phat, N, lambda_1, lambda_2):
"""
a: (N,) number of measurements at that timestep
Phat: (N, 3) sum of measurements at that timestep
N: num time steps
lambda_1, lambda_2: regularization parameters
solves the optimization problem (over P \in R^{Tx3}):
minimize ||diag(a)*P-Phat||^2 + lambda_1/N*||D_2*P||^2 + lambda_2/N*||D_3*P||^2
returns:
- P: (N, 3) matrix with full trajectory
"""
# A in Banded Matrix form
A = bm.diag(1. * a)
# D_2 and D_3 in Banded Matrix form transposed
D_2_bm_T = bm.BandMat(
1, 1,
np.hstack([
np.zeros((3, 1)),
np.repeat([[1.], [-2.], [1.]], N - 2, axis=1),
np.zeros((3, 1))
]))
D_3_bm_T = bm.BandMat(
2, 2,
np.hstack([
np.zeros((5, 2)),
np.repeat([[-1.], [2.], [0.], [-2.], [1.]], N - 4, axis=1),
np.zeros((5, 2))
]))
# XP=B normal equations
X = bm.dot_mm(A, A) + lambda_1 / N * bm.dot_mm(
D_2_bm_T, D_2_bm_T.T) + lambda_2 / N * bm.dot_mm(D_3_bm_T, D_3_bm_T.T)
l_and_u = (X.l, X.u) # lower and upper band bounds
B = np.hstack([
np.expand_dims(bm.dot_mv(A, Phat[:, 0]), -1),
np.expand_dims(bm.dot_mv(A, Phat[:, 1]), -1),
np.expand_dims(bm.dot_mv(A, Phat[:, 2]), -1)
])
# solve normal equations
P = solve_banded(l_and_u, X.data, B)
return P
def test__cholesky_banded(self, its=100):
for it in range(its):
size = random.choice([0, 1, randint(0, 10), randint(0, 100)])
if rand_bool():
mat_bm = gen_pos_def_BandMat(size, transposed=False)
else:
mat_bm = gen_symmetric_BandMat(size, transposed=False)
# make it a bit more likely to be pos def
bm.diag(mat_bm)[:] = np.abs(bm.diag(mat_bm)) + 0.1
depth = mat_bm.l
lower = rand_bool()
if lower:
mat_half_data = mat_bm.data[depth:]
else:
mat_half_data = mat_bm.data[:(depth + 1)]
overwrite = rand_bool()
mat_half_data_arg = mat_half_data.copy()
try:
chol_data = bla._cholesky_banded(mat_half_data_arg,
overwrite_ab=overwrite,
lower=lower)
except la.LinAlgError as e:
# First part of the message is e.g. "2-th leading minor".
msgRe = (r'^' + re.escape(str(e)[:15]) +
r'.*not positive definite$')
with self.assertRaisesRegexp(la.LinAlgError, msgRe):
sla.cholesky(mat_bm.full(), lower=lower)
else:
assert np.shape(chol_data) == (depth + 1, size)
if lower:
chol_bm = bm.BandMat(depth, 0, chol_data)
mat_bm_again = bm.dot_mm(chol_bm, chol_bm.T)
else:
chol_bm = bm.BandMat(0, depth, chol_data)
mat_bm_again = bm.dot_mm(chol_bm.T, chol_bm)
assert_allclose(mat_bm_again.full(), mat_bm.full())
if size > 0:
self.assertEqual(
np.may_share_memory(chol_data, mat_half_data_arg),
overwrite)
if not overwrite:
assert np.all(mat_half_data_arg == mat_half_data)
def test_cholesky(self, its=50):
for it in range(its):
size = random.choice([0, 1, randint(0, 10), randint(0, 100)])
mat_bm = gen_pos_def_BandMat(size)
depth = mat_bm.l
lower = rand_bool()
alternative = rand_bool()
chol_bm = bla.cholesky(mat_bm, lower=lower,
alternative=alternative)
assert chol_bm.l == (depth if lower else 0)
assert chol_bm.u == (0 if lower else depth)
assert not np.may_share_memory(chol_bm.data, mat_bm.data)
if lower != alternative:
mat_bm_again = bm.dot_mm(chol_bm, chol_bm.T)
else:
mat_bm_again = bm.dot_mm(chol_bm.T, chol_bm)
assert_allclose(mat_bm_again.full(), mat_bm.full())
def test_cholesky(self, its=50):
for it in range(its):
size = random.choice([0, 1, randint(0, 10), randint(0, 100)])
mat_bm = gen_pos_def_BandMat(size)
depth = mat_bm.l
lower = rand_bool()
alternative = rand_bool()
chol_bm = bla.cholesky(mat_bm, lower=lower,
alternative=alternative)
assert chol_bm.l == (depth if lower else 0)
assert chol_bm.u == (0 if lower else depth)
assert not np.may_share_memory(chol_bm.data, mat_bm.data)
if lower != alternative:
mat_bm_again = bm.dot_mm(chol_bm, chol_bm.T)
else:
mat_bm_again = bm.dot_mm(chol_bm.T, chol_bm)
assert_allclose(mat_bm_again.full(), mat_bm.full())
def test_dot_mm(self, its=50):
for it in range(its):
size = random.choice([0, 1, randint(0, 10), randint(0, 100)])
a_bm = gen_BandMat(size)
b_bm = gen_BandMat(size)
diag = None if rand_bool() else randn(size)
diag_value = np.ones((size,)) if diag is None else diag
a_full = a_bm.full()
b_full = b_bm.full()
c_bm = bm.dot_mm(a_bm, b_bm, diag=diag)
c_full = np.dot(np.dot(a_full, np.diag(diag_value)), b_full)
assert c_bm.l == a_bm.l + b_bm.l
assert c_bm.u == a_bm.u + b_bm.u
assert c_bm.size == size
assert_allclose(c_bm.full(), c_full)
assert not np.may_share_memory(c_bm.data, a_bm.data)
assert not np.may_share_memory(c_bm.data, b_bm.data)
if diag is not None:
assert not np.may_share_memory(c_bm.data, diag)
def test_dot_mm(self, its=50):
for it in range(its):
size = random.choice([0, 1, randint(0, 10), randint(0, 100)])
a_bm = gen_BandMat(size)
b_bm = gen_BandMat(size)
diag = None if rand_bool() else randn(size)
diag_value = np.ones((size, )) if diag is None else diag
a_full = a_bm.full()
b_full = b_bm.full()
c_bm = bm.dot_mm(a_bm, b_bm, diag=diag)
c_full = np.dot(np.dot(a_full, np.diag(diag_value)), b_full)
assert c_bm.l == a_bm.l + b_bm.l
assert c_bm.u == a_bm.u + b_bm.u
assert c_bm.size == size
assert_allclose(c_bm.full(), c_full)
assert not np.may_share_memory(c_bm.data, a_bm.data)
assert not np.may_share_memory(c_bm.data, b_bm.data)
if diag is not None:
assert not np.may_share_memory(c_bm.data, diag)
a_bm = bm.BandMat(
1, 1,
np.array([
[0.0, 0.2, 0.3, 0.4, 0.5],
[1.0, 0.9, 1.1, 0.8, 1.3],
[0.3, 0.1, 0.5, 0.6, 0.0],
]))
b_bm = bm.BandMat(
1, 0, np.array([
[1.0, 0.9, 1.1, 0.8, 1.3],
[0.3, 0.1, 0.5, 0.6, 0.0],
]))
c_bm = bm.dot_mm(a_bm.T, b_bm)
d_bm = a_bm + b_bm
a_full = a_bm.full()
b_full = b_bm.full()
c_full = c_bm.full()
d_full = d_bm.full()
print('a_full:')
print(a_full)
print()
print('b_full:')
print(b_full)
print()
print('np.dot(a_full.T, b_full):')
def solve_band(refl, solve_edge=True, W=None, alpha=1000, unc=False, drop=True):
"""
test run function
"""
Iobs = (refl>0)
nObs = (Iobs).sum(axis=0)
# scale alpha
alpha = 1
alpha = alpha * np.nanmean(nObs)
alpha = np.clip(alpha, 20, 250)
if solve_edge:
T = .6 * alpha
nT, ys, xs = refl.shape
# make D matrix
I = np.eye(nT) #np.diag (np.ones(nx))
D = (I - np.roll(I, -1)).T
#D = D.T.dot(D)
D1A = np.zeros((nT, 2))
D1A[1:, 0] = np.diag(D, 1)
D1A[:, 1] = np.diag(D, 0)
# convert to banded matrices
_D = bm.BandMat(0, 1, D1A.T, transposed=False)
# create mats
A, B, C, D = prepare_mats(refl, alpha=alpha)
# Initial run with no smoothing
X = TDMA_MAT(A, B, C, D)
W0 = np.ones_like(X)*100
WW = np.ones_like(X)
X0 = np.zeros_like(D)
CONV = np.zeros((ys, xs)).astype(np.bool)
Nits = np.zeros((ys, xs), dtype=np.int)
for i in range(5):
# calculate weights
order_n = 1
#dx = np.diff(X, axis=0, n=order_n)* alpha**2
dx = np.gradient(X, axis=0) * alpha**2
# enforce direction constraint
if drop != None:
if drop:
dx[dx>0]=0
else:
dx[dx<=0]=0
# eg frst 30 days and last 30 days
_w = np.exp(-dx**2 / T)
"""
Scaling from CERC
"""
y,x=30, 57
wScale = nObs / np.sum(_w, axis=0)
_w = _w * wScale
_w = np.clip(_w, 0., 1)
np.place(_w, np.isnan(_w), 1)
WW[:]=_w
"""
Update the regularisation matrix
with the weights
-- This involves a dot product
with a diag eg D.T W D
so i don't how to do this atm
This is the slowest so if this can be done fast much better
"""
for y in range(ys):
for x in range(xs):
if not CONV[y,x]:
# no convergence keep going
n = alpha * bm.dot_mm(_D.T, _D, diag=WW[:, y, x].astype(np.float))
a = n.data[2][:-1]
b = n.data[1] + Iobs[:, y, x]
c = n.data[0][1:]
# update matrices
A[:, y, x]=a
B[:, y, x]=b
C[:, y, x]=c
Nits[y,x]+=1
"""
Run again with new weights
"""
X = TDMA_MAT(A, B, C, D)
"""
assess convergence
-eg are the weights still changing
"""
cow = np.sum(np.abs((WW - W0) / W0), axis=0)+ 1e-6
CONV[cow<1e-2]=True
MINITER=3
if i< MINITER:
CONV[:]=False
CONV[WW.min(axis=0)<0.1]=True
W0 = WW
X0 = X
"""
Want to check steps and re-inforce real steps
"""
W, keptEdges, Sch = refine_edges(WW, X, refl)
"""
Resolve with refined edges
"""
for y in range(ys):
for x in range(xs):
n = alpha * bm.dot_mm(_D.T, _D, diag=W[:, y, x].astype(np.float))
a = n.data[2][:-1]
b = n.data[1] + Iobs[:, y, x]
c = n.data[0][1:]
# update matrices
A[:, y, x]=a
B[:, y, x]=b
C[:, y, x]=c
Nits[y,x]+=1
"""
Run again with the refined weights
"""
#y,x=43, 62
if unc:
X = TDMA_MAT(A, B, C, D)
DD = np.copy(D).astype(np.float32)
Inv = np.zeros_like(X)
# only need the middle month tbh
for t in range(20, nT-20):
DD[:]=0
DD[t]=1
Inv[t]=TDMA_MAT(A, B, C, DD)[t]
return X, Inv, W, CONV, Nits, Sch
else:
X = TDMA_MAT(A, B, C, D)
return X, W, CONV, Nits, Sch
else:
"""
Edge has been provided so use that
"""
Iobs = (refl>0)
nObs = (Iobs).sum(axis=0)
nT, ys, xs = refl.shape
# make D matrix
I = np.eye(nT) #np.diag (np.ones(nx))
D = (I - np.roll(I, -1)).T
#D = D.T.dot(D)
D1A = np.zeros((nT, 2))
D1A[1:, 0] = np.diag(D, 1)
D1A[:, 1] = np.diag(D, 0)
# convert to banded matrices
_D = bm.BandMat(0, 1, D1A.T, transposed=False)
# create mats
A, B, C, D = prepare_mats(refl, alpha=alpha)
# add the weights matrix
for y in range(ys):
for x in range(xs):
n = alpha * bm.dot_mm(_D.T, _D, diag=W[:, y, x].astype(np.float))
a = n.data[2][:-1]
b = n.data[1] + Iobs[:, y, x]
c = n.data[0][1:]
# update matrices
A[:, y, x]=a
B[:, y, x]=b
C[:, y, x]=c
"""
Run again with the refined weights
"""
X = TDMA_MAT(A, B, C, D)
return X, W
def unit_variance_mlpg_matrix(windows, T):
"""Compute MLPG matrix assuming input is normalized to have unit-variances.
Let :math:`\mu` is the input mean sequence (``num_windows*T x static_dim``),
:math:`W` is a window matrix ``(T x num_windows*T)``, assuming input is
normalized to have unit-variances, MLPG can be written as follows:
.. math::
y = R \mu
where
.. math::
R = (W^{T} W)^{-1} W^{T}
Here we call :math:`R` as the MLPG matrix.
Args:
windows: (list): List of windows.
T (int): Number of frames.
Returns:
numpy.ndarray: MLPG matrix (``T x nun_windows*T``).
See also:
:func:`nnmnkwii.autograd.UnitVarianceMLPG`,
:func:`nnmnkwii.paramgen.mlpg`.
Examples:
>>> from nnmnkwii import paramgen as G
>>> import numpy as np
>>> windows = [
... (0, 0, np.array([1.0])),
... (1, 1, np.array([-0.5, 0.0, 0.5])),
... (1, 1, np.array([1.0, -2.0, 1.0])),
... ]
>>> G.unit_variance_mlpg_matrix(windows, 3)
array([[ 2.73835927e-01, 1.95121944e-01, 9.20177400e-02,
9.75609720e-02, -9.09090936e-02, -9.75609720e-02,
-3.52549881e-01, -2.43902430e-02, 1.10864742e-02],
[ 1.95121944e-01, 3.41463417e-01, 1.95121944e-01,
1.70731708e-01, -5.55111512e-17, -1.70731708e-01,
-4.87804860e-02, -2.92682916e-01, -4.87804860e-02],
[ 9.20177400e-02, 1.95121944e-01, 2.73835927e-01,
9.75609720e-02, 9.09090936e-02, -9.75609720e-02,
1.10864742e-02, -2.43902430e-02, -3.52549881e-01]], dtype=float32)
"""
win_mats = build_win_mats(windows, T)
sdw = np.max([win_mat.l + win_mat.u for win_mat in win_mats])
max_win_width = np.max([max(win_mat.l, win_mat.u) for win_mat in win_mats])
P = bm.zeros(sdw, sdw, T)
# set edge precitions to zero
precisions = bm.zeros(0, 0, T)
precisions.data[:, max_win_width:-max_win_width] += 1.0
mod_win_mats = []
for win_index, win_mat in enumerate(win_mats):
if win_index != 0:
# use zero precisions for dynamic features
mod_win_mat = bm.dot_mm(precisions, win_mat)
bm.dot_mm_plus_equals(mod_win_mat.T, win_mat, target_bm=P)
mod_win_mats.append(mod_win_mat)
else:
# static features
bm.dot_mm_plus_equals(win_mat.T, win_mat, target_bm=P)
mod_win_mats.append(win_mat)
chol_bm = bla.cholesky(P, lower=True)
Pinv = cholesky_inv_banded(chol_bm.full(), width=chol_bm.l + chol_bm.u + 1)
cocatenated_window = full_window_mat(mod_win_mats, T)
return Pinv.dot(cocatenated_window.T).astype(np.float32)
def mlpg_grad(mean_frames, variance_frames, windows, grad_output):
"""MLPG gradient computation
Parameters are same as :func:`nnmnkwii.paramgen.mlpg` except for
``grad_output``. See the function docmenent for what the parameters mean.
Let :math:`d` is the index of static features, :math:`l` is the index
of windows, gradients :math:`g_{d,l}` can be computed by:
.. math::
g_{d,l} = (\sum_{l} W_{l}^{T}P_{d,l}W_{l})^{-1} W_{l}^{T}P_{d,l}
where :math:`W_{l}` is a banded window matrix and :math:`P_{d,l}` is a
diagonal precision matrix.
Assuming the variances are diagonals, MLPG can be performed in
dimention-by-dimention efficiently.
Let :math:`o_{d}` be ``T`` dimentional back-propagated gradients, the
resulting gradients :math:`g'_{l,d}` to be propagated are
computed as follows:
.. math::
g'_{d,l} = o_{d}^{T} g_{d,l}
Args:
mean_frames (numpy.ndarray): Means.
variance_frames (numpy.ndarray): Variances.
windows (list): Windows.
grad_output: Backpropagated output gradient, shape (``T x static_dim``)
Returns:
numpy.ndarray: Gradients to be back propagated, shape: (``T x D``)
See also:
:func:`nnmnkwii.autograd.mlpg`, :class:`nnmnkwii.autograd.MLPG`
"""
T, D = mean_frames.shape
win_mats = build_win_mats(windows, T)
static_dim = D // len(windows)
max_win_width = np.max([max(win_mat.l, win_mat.u) for win_mat in win_mats])
grads = np.zeros((T, D), dtype=np.float32)
for d in range(static_dim):
sdw = max([win_mat.l + win_mat.u for win_mat in win_mats])
# R: \sum_{l} W_{l}^{T}P_{d,l}W_{l}
R = bm.zeros(sdw, sdw, T) # overwritten in the loop
# dtype = np.float64 for bandmat
precisions = np.zeros((len(windows), T), dtype=np.float64)
for win_idx, win_mat in enumerate(win_mats):
precisions[win_idx] = 1 / \
variance_frames[:, win_idx * static_dim + d]
# use zero precisions at edge frames for dynamic features
if win_idx != 0:
precisions[win_idx, :max_win_width] = 0
precisions[win_idx, -max_win_width:] = 0
bm.dot_mm_plus_equals(win_mat.T,
win_mat,
target_bm=R,
diag=precisions[win_idx])
for win_idx, win_mat in enumerate(win_mats):
# r: W_{l}^{T}P_{d,l}
r = bm.dot_mm(win_mat.T, bm.diag(precisions[win_idx]))
# grad_{d, l} = R^{-1r}
grad = solve_banded((R.l, R.u), R.data, r.full())
assert grad.shape == (T, T)
# Finally we get grad for a particular dimension
grads[:, win_idx * static_dim + d] = grad_output[:, d].T.dot(grad)
return grads
np.array([
[0.0, 0.2, 0.3, 0.4, 0.5],
[1.0, 0.9, 1.1, 0.8, 1.3],
[0.3, 0.1, 0.5, 0.6, 0.0],
])
)
b_bm = bm.BandMat(
1, 0,
np.array([
[1.0, 0.9, 1.1, 0.8, 1.3],
[0.3, 0.1, 0.5, 0.6, 0.0],
])
)
c_bm = bm.dot_mm(a_bm.T, b_bm)
d_bm = a_bm + b_bm
a_full = a_bm.full()
b_full = b_bm.full()
c_full = c_bm.full()
d_full = d_bm.full()
print('a_full:')
print(a_full)
print()
print('b_full:')
print(b_full)
print()
print('np.dot(a_full.T, b_full):')
def solve_banded_fast(y,
alpha,
band=1,
do_edges=True,
W=None,
rmse=0.1,
_D=None):
"""
Speed up to actually solve the thing really fast
using
scipy.linalg.solve_banded(L.T, y)
"""
# do some prep work
nT = y.shape[0]
q = np.zeros(nT)
eidx = y > 0
q[eidx] = 1
Iobs = q
# unc
#Iobs /= C_obs[band]
#y /= C_obs[band]
nObs = eidx.sum()
"""
*-- Alpha value --*
This needs to be normalised by
the number of observations
OR DO I ??
"""
"""
make D matrix
in flattened form
faster to pre-compute as is
it's a standard length
"""
#_D = None
if _D == None:
D = make_D_matrix(nT)
else:
D = _D
"""
get a smooth solution
"""
# weighting for outliers
Wr = np.ones(nT)
scale = (nObs)
# re-scale alpha
"""
re-scale alpha
So trying something a little
different...
Basically allow this to vary across
the time-series too based on the number of samples
in a local window. I think this should improve the
scaling of alpha so we can keep edges where there
are lots of samples....
"""
#winN = np.ones(20)
#nScale = np.convolve(eidx, winN, mode='same')
#alpha = 0.5*alpha * nScale + 0.5*alpha * scale
alpha = alpha * scale
alpha = np.minimum(alpha, 200)
alpha = np.maximum(alpha, 70)
D2 = bm.dot_mm(D.T, D)
dl = alpha * D2.data[2][:-1]
di = alpha * D2.data[1] + Iobs * Wr
du = alpha * D2.data[0][1:]
ret = GTSV(dl, di, du, y, False, False, False, False)
x0 = ret[-2]
#import pdb; pdb.set_trace()
"""
*-- now do edge preserving --*
"""
cost = []
tx = np.arange(len(y))[eidx]
if do_edges:
# Define some stuff
converged = False
convT = 1e-5
itera = 0
MAXITER = 10
MINITER = 8
T = 0.01 # threshold in the change needed...
# Define the weights matrix
_wg = np.ones(nT)
W = bm.diag(_wg)
# run until convergence
x = x0
R = []
co0 = 1e3
var_C = C_obs[band]
sig_C = np.sqrt(var_C)
obs = y[eidx]
_w = np.ones(nT - 1)
_w0 = np.ones(nT - 1) * 100
while not converged and itera < MAXITER:
"""
Robust smoothing...
using the method from the garcia
dct paper
"""
r = x[eidx] - y[eidx]
"""
h is the leverage from the
hat matrix ege the inverse matrix
"""
h = 0.0009
"""
Robust estimate of the standard
deviation of residuals
"""
#sig_rob = 1.4826 * np.median(r - np.median(r))
ui = r / (sig_C)
#Wr[eidx] = (1-(ui/4.685)**2)**2
# mask bad
#bad = np.abs(ui/4.685)>2
#Wr[eidx][bad]=0
"""
*- edge preserving --*
So this should now be fixed.
The functional is
w = T / sqrt( T**2 + (dx * alpha**2)**2 )
"""
"""
The derivative has to be scale by
alpha
"""
#dx = np.diff(x) * alpha**2
#ww[1:] = T/(np.sqrt(T**2 + dx**2))
dx = np.diff(x) * alpha**2
# only want drop in NIR
dx[dx > 0] = 0
"""
Thereshol T has to be scaled relative
to alpha (alpha actually works well)
"""
T = 0.7 * alpha
# calulcate the edge preserving functional w
_w = T / np.sqrt(T**2 + dx**2)
_w = np.exp(-dx**2 / T**2)
"""
Scaling from CERC
"""
wScale = nObs / np.sum(_w)
_w = _w * wScale
#_w = scipy.ndimage.minimum_filter(_w, 8)
co1 = np.sqrt(np.sum((x0[eidx] - obs)**2 / var_C) / nObs)
# faster version
cost.append(co1)
# remove nan
if itera > 2:
"""
Allow for some initial robust
outlier rejection
"""
np.place(_w, np.isnan(_w), 1)
W.data[0, :-1] = _w
else:
_w[:] = 1
"""
Remove edge effects
-- force every outside
in the edge windows back to 1
"""
W.data[0, :5] = 1
W.data[0, -5:] = 1
"""
I cant figure out the flat formulat to do below
"""
reg = alpha * bm.dot_mm(bm.dot_mm(D.T, W), D)
dl = reg.data[2][:-1]
di = reg.data[1] + Iobs * Wr
du = reg.data[0][1:]
ret = GTSV(dl, di, du, y, False, False, False, False)
x = ret[-2]
co = np.sum((x - x0)**2) / np.sum(x0**2) + 1e-6
cow = np.sum((_w - _w0)**2) / np.sum(_w0**2) + 1e-6
#if co < convT:
# converged=True
if cow < 1e5:
converged = True
if np.abs(co1 - co0) < 0.01:
converged = True
if itera < MINITER:
converged = False
if _w.min() < 0.1:
converged = True
x0 = x
_w0 = _w
co0 = co1
itera += 1
"""
Do some checking
So some times the algorithm freaks out
and over does the edge preserving...
Let's do some checks
1. Want one strong area with a small w
If w meets certain conditions optimise it a bit
- Sparsify it..
eg take the min and keep one min
place the min at the end of its window
"""
if np.any(_w < 0.3):
"""
sufficent edge process
to check whats going on
"""
"""
Check 1.
What percentage of timesteps are below
a threhsold --> eg is w all over the place?
and we've added to many edges
"""
reset = False
frac = (_w < 0.3).sum() / float(_w.shape[0])
# want no more than 5%
if frac > 0.1:
# re-set edge process
_w.fill(1)
reset = True
idx = np.nanargmin(_w)
val = _w[idx]
"""
Want to check a couple of things
1. If min w aligns with an observation
how far off the prediction are we?
--> this helps remove cases where
we have shot noise from clouds
or cloud shadows
"""
if eidx[idx]: # have an observ
if np.abs((y[idx] - x[idx])) / np.sqrt(C_obs[band]) > 2:
# it's a nonse break remove
_w.fill(1)
reset = True
else:
# check the surrounding pixels
# to find some obs to check
low = np.maximum(idx - 40, 0)
upp = np.minimum(idx + 40, 363)
mask = eidx[low:upp]
yy = y[low:upp][mask]
xx = x[low:upp][mask]
_ww = _w[low:upp][mask]
zz = np.abs((yy - xx)) / np.sqrt(C_obs[band])
cond = np.logical_and(_ww < 0.3, zz > 2).sum()
if cond > 4:
_w.fill(1)
reset = True
"""
Resolve it...
"""
if not reset:
"""
So this is probably a step
fix it up a bit
"""
#_w[:]=1
_w[idx] = 0.01
pass
W.data[0, :-1] = _w
"""
I cant figure out the flat formulat to do below
"""
reg = alpha * bm.dot_mm(bm.dot_mm(D.T, W), D)
dl = reg.data[2][:-1]
di = reg.data[1] + Iobs
du = reg.data[0][1:]
ret = GTSV(dl, di, du, y, False, False, False, False)
x = ret[-2]
else:
# use the smooth solution
x = x0
if type(W) != None and do_edges == False:
"""
Weights matrix supplied already...
use this for edges...
"""
# make W into a bandmat matrix
W = bm.diag(W)
reg = alpha * bm.dot_mm(bm.dot_mm(D.T, W), D)
dl = reg.data[2][:-1]
di = reg.data[1] + Iobs
du = reg.data[0][1:]
ret = GTSV(dl, di, du, y, False, False, False, False)
x = ret[-2]
"""
Add the MSE (eg obs unc estimate) to inflat unc...
"""
C = GTSV(dl, di, du, np.eye(nT), False, False, False, False)[-2]
unc = np.sqrt(C_obs[band] * np.diag(C))
return x, unc, W.data.flatten(), D
