def test__cholesky_banded(self, its=100):
for it in range(its):
size = random.choice([0, 1, randint(0, 10), randint(0, 100)])
if rand_bool():
mat_bm = gen_pos_def_BandMat(size, transposed=False)
else:
mat_bm = gen_symmetric_BandMat(size, transposed=False)
# make it a bit more likely to be pos def
bm.diag(mat_bm)[:] = np.abs(bm.diag(mat_bm)) + 0.1
depth = mat_bm.l
lower = rand_bool()
if lower:
mat_half_data = mat_bm.data[depth:]
else:
mat_half_data = mat_bm.data[:(depth + 1)]
overwrite = rand_bool()
mat_half_data_arg = mat_half_data.copy()
try:
chol_data = bla._cholesky_banded(
mat_half_data_arg, overwrite_ab=overwrite, lower=lower
)
except la.LinAlgError as e:
# First part of the message is e.g. "2-th leading minor".
msgRe = (r'^' + re.escape(str(e)[:15]) +
r'.*not positive definite$')
with self.assertRaisesRegexp(la.LinAlgError, msgRe):
sla.cholesky(mat_bm.full(), lower=lower)
else:
assert np.shape(chol_data) == (depth + 1, size)
if lower:
chol_bm = bm.BandMat(depth, 0, chol_data)
mat_bm_again = bm.dot_mm(chol_bm, chol_bm.T)
else:
chol_bm = bm.BandMat(0, depth, chol_data)
mat_bm_again = bm.dot_mm(chol_bm.T, chol_bm)
assert_allclose(mat_bm_again.full(), mat_bm.full())
if size > 0:
self.assertEqual(
np.may_share_memory(chol_data, mat_half_data_arg),
overwrite
)
if not overwrite:
assert np.all(mat_half_data_arg == mat_half_data)
def test__cholesky_banded(self, its=100):
for it in range(its):
size = random.choice([0, 1, randint(0, 10), randint(0, 100)])
if rand_bool():
mat_bm = gen_pos_def_BandMat(size, transposed=False)
else:
mat_bm = gen_symmetric_BandMat(size, transposed=False)
# make it a bit more likely to be pos def
bm.diag(mat_bm)[:] = np.abs(bm.diag(mat_bm)) + 0.1
depth = mat_bm.l
lower = rand_bool()
if lower:
mat_half_data = mat_bm.data[depth:]
else:
mat_half_data = mat_bm.data[:(depth + 1)]
overwrite = rand_bool()
mat_half_data_arg = mat_half_data.copy()
try:
chol_data = bla._cholesky_banded(mat_half_data_arg,
overwrite_ab=overwrite,
lower=lower)
except la.LinAlgError as e:
# First part of the message is e.g. "2-th leading minor".
msgRe = (r'^' + re.escape(str(e)[:15]) +
r'.*not positive definite$')
with self.assertRaisesRegexp(la.LinAlgError, msgRe):
sla.cholesky(mat_bm.full(), lower=lower)
else:
assert np.shape(chol_data) == (depth + 1, size)
if lower:
chol_bm = bm.BandMat(depth, 0, chol_data)
mat_bm_again = bm.dot_mm(chol_bm, chol_bm.T)
else:
chol_bm = bm.BandMat(0, depth, chol_data)
mat_bm_again = bm.dot_mm(chol_bm.T, chol_bm)
assert_allclose(mat_bm_again.full(), mat_bm.full())
if size > 0:
self.assertEqual(
np.may_share_memory(chol_data, mat_half_data_arg),
overwrite)
if not overwrite:
assert np.all(mat_half_data_arg == mat_half_data)
def test_diag(self, its=50):
for it in range(its):
size = random.choice([0, 1, randint(0, 10), randint(0, 100)])
vec = randn(size)
mat_bm = bm.diag(vec)
assert isinstance(mat_bm, bm.BandMat)
assert_allequal(mat_bm.full(), np.diag(vec))
assert mat_bm.data.base is vec
if size > 0:
assert np.may_share_memory(mat_bm.data, vec)
mat_bm = gen_BandMat(size)
vec = bm.diag(mat_bm)
assert_allequal(vec, np.diag(mat_bm.full()))
assert vec.base is mat_bm.data
if size > 0:
assert np.may_share_memory(vec, mat_bm.data)
def test_diag(self, its=50):
for it in range(its):
size = random.choice([0, 1, randint(0, 10), randint(0, 100)])
vec = randn(size)
mat_bm = bm.diag(vec)
assert isinstance(mat_bm, bm.BandMat)
assert_allequal(mat_bm.full(), np.diag(vec))
assert mat_bm.data.base is vec
if size > 0:
assert np.may_share_memory(mat_bm.data, vec)
mat_bm = gen_BandMat(size)
vec = bm.diag(mat_bm)
assert_allequal(vec, np.diag(mat_bm.full()))
assert vec.base is mat_bm.data
if size > 0:
assert np.may_share_memory(vec, mat_bm.data)
def test_solve(self, its=50):
for it in range(its):
size = random.choice([0, 1, randint(0, 10), randint(0, 100)])
b = randn(size)
# the below tries to ensure the matrix is well-conditioned
a_bm = gen_BandMat(size) + bm.diag(np.ones((size,)) * 10.0)
a_full = a_bm.full()
x = bla.solve(a_bm, b)
assert_allclose(bm.dot_mv(a_bm, x), b)
if size == 0:
x_good = np.zeros((size,))
else:
x_good = sla.solve(a_full, b)
assert_allclose(x, x_good)
assert not np.may_share_memory(x, a_bm.data)
assert not np.may_share_memory(x, b)
def test_solve(self, its=50):
for it in range(its):
size = random.choice([0, 1, randint(0, 10), randint(0, 100)])
b = randn(size)
# the below tries to ensure the matrix is well-conditioned
a_bm = gen_BandMat(size) + bm.diag(np.ones((size,)) * 10.0)
a_full = a_bm.full()
x = bla.solve(a_bm, b)
assert_allclose(bm.dot_mv(a_bm, x), b)
if size == 0:
x_good = np.zeros((size,))
else:
x_good = sla.solve(a_full, b)
assert_allclose(x, x_good)
assert not np.may_share_memory(x, a_bm.data)
assert not np.may_share_memory(x, b)
def smooth(a, Phat, N, lambda_1, lambda_2):
"""
a: (N,) number of measurements at that timestep
Phat: (N, 3) sum of measurements at that timestep
N: num time steps
lambda_1, lambda_2: regularization parameters
solves the optimization problem (over P \in R^{Tx3}):
minimize ||diag(a)*P-Phat||^2 + lambda_1/N*||D_2*P||^2 + lambda_2/N*||D_3*P||^2
returns:
- P: (N, 3) matrix with full trajectory
"""
# A in Banded Matrix form
A = bm.diag(1. * a)
# D_2 and D_3 in Banded Matrix form transposed
D_2_bm_T = bm.BandMat(
1, 1,
np.hstack([
np.zeros((3, 1)),
np.repeat([[1.], [-2.], [1.]], N - 2, axis=1),
np.zeros((3, 1))
]))
D_3_bm_T = bm.BandMat(
2, 2,
np.hstack([
np.zeros((5, 2)),
np.repeat([[-1.], [2.], [0.], [-2.], [1.]], N - 4, axis=1),
np.zeros((5, 2))
]))
# XP=B normal equations
X = bm.dot_mm(A, A) + lambda_1 / N * bm.dot_mm(
D_2_bm_T, D_2_bm_T.T) + lambda_2 / N * bm.dot_mm(D_3_bm_T, D_3_bm_T.T)
l_and_u = (X.l, X.u) # lower and upper band bounds
B = np.hstack([
np.expand_dims(bm.dot_mv(A, Phat[:, 0]), -1),
np.expand_dims(bm.dot_mv(A, Phat[:, 1]), -1),
np.expand_dims(bm.dot_mv(A, Phat[:, 2]), -1)
])
# solve normal equations
P = solve_banded(l_and_u, X.data, B)
return P
def mlpg_grad(mean_frames, variance_frames, windows, grad_output):
"""MLPG gradient computation
Parameters are same as :func:`nnmnkwii.paramgen.mlpg` except for
``grad_output``. See the function docmenent for what the parameters mean.
Let :math:`d` is the index of static features, :math:`l` is the index
of windows, gradients :math:`g_{d,l}` can be computed by:
.. math::
g_{d,l} = (\sum_{l} W_{l}^{T}P_{d,l}W_{l})^{-1} W_{l}^{T}P_{d,l}
where :math:`W_{l}` is a banded window matrix and :math:`P_{d,l}` is a
diagonal precision matrix.
Assuming the variances are diagonals, MLPG can be performed in
dimention-by-dimention efficiently.
Let :math:`o_{d}` be ``T`` dimentional back-propagated gradients, the
resulting gradients :math:`g'_{l,d}` to be propagated are
computed as follows:
.. math::
g'_{d,l} = o_{d}^{T} g_{d,l}
Args:
mean_frames (numpy.ndarray): Means.
variance_frames (numpy.ndarray): Variances.
windows (list): Windows.
grad_output: Backpropagated output gradient, shape (``T x static_dim``)
Returns:
numpy.ndarray: Gradients to be back propagated, shape: (``T x D``)
See also:
:func:`nnmnkwii.autograd.mlpg`, :class:`nnmnkwii.autograd.MLPG`
"""
T, D = mean_frames.shape
win_mats = build_win_mats(windows, T)
static_dim = D // len(windows)
max_win_width = np.max([max(win_mat.l, win_mat.u) for win_mat in win_mats])
grads = np.zeros((T, D), dtype=np.float32)
for d in range(static_dim):
sdw = max([win_mat.l + win_mat.u for win_mat in win_mats])
# R: \sum_{l} W_{l}^{T}P_{d,l}W_{l}
R = bm.zeros(sdw, sdw, T) # overwritten in the loop
# dtype = np.float64 for bandmat
precisions = np.zeros((len(windows), T), dtype=np.float64)
for win_idx, win_mat in enumerate(win_mats):
precisions[win_idx] = 1 / \
variance_frames[:, win_idx * static_dim + d]
# use zero precisions at edge frames for dynamic features
if win_idx != 0:
precisions[win_idx, :max_win_width] = 0
precisions[win_idx, -max_win_width:] = 0
bm.dot_mm_plus_equals(win_mat.T,
win_mat,
target_bm=R,
diag=precisions[win_idx])
for win_idx, win_mat in enumerate(win_mats):
# r: W_{l}^{T}P_{d,l}
r = bm.dot_mm(win_mat.T, bm.diag(precisions[win_idx]))
# grad_{d, l} = R^{-1r}
grad = solve_banded((R.l, R.u), R.data, r.full())
assert grad.shape == (T, T)
# Finally we get grad for a particular dimension
grads[:, win_idx * static_dim + d] = grad_output[:, d].T.dot(grad)
return grads
def solve_banded_fast(y,
alpha,
band=1,
do_edges=True,
W=None,
rmse=0.1,
_D=None):
"""
Speed up to actually solve the thing really fast
using
scipy.linalg.solve_banded(L.T, y)
"""
# do some prep work
nT = y.shape[0]
q = np.zeros(nT)
eidx = y > 0
q[eidx] = 1
Iobs = q
# unc
#Iobs /= C_obs[band]
#y /= C_obs[band]
nObs = eidx.sum()
"""
*-- Alpha value --*
This needs to be normalised by
the number of observations
OR DO I ??
"""
"""
make D matrix
in flattened form
faster to pre-compute as is
it's a standard length
"""
#_D = None
if _D == None:
D = make_D_matrix(nT)
else:
D = _D
"""
get a smooth solution
"""
# weighting for outliers
Wr = np.ones(nT)
scale = (nObs)
# re-scale alpha
"""
re-scale alpha
So trying something a little
different...
Basically allow this to vary across
the time-series too based on the number of samples
in a local window. I think this should improve the
scaling of alpha so we can keep edges where there
are lots of samples....
"""
#winN = np.ones(20)
#nScale = np.convolve(eidx, winN, mode='same')
#alpha = 0.5*alpha * nScale + 0.5*alpha * scale
alpha = alpha * scale
alpha = np.minimum(alpha, 200)
alpha = np.maximum(alpha, 70)
D2 = bm.dot_mm(D.T, D)
dl = alpha * D2.data[2][:-1]
di = alpha * D2.data[1] + Iobs * Wr
du = alpha * D2.data[0][1:]
ret = GTSV(dl, di, du, y, False, False, False, False)
x0 = ret[-2]
#import pdb; pdb.set_trace()
"""
*-- now do edge preserving --*
"""
cost = []
tx = np.arange(len(y))[eidx]
if do_edges:
# Define some stuff
converged = False
convT = 1e-5
itera = 0
MAXITER = 10
MINITER = 8
T = 0.01 # threshold in the change needed...
# Define the weights matrix
_wg = np.ones(nT)
W = bm.diag(_wg)
# run until convergence
x = x0
R = []
co0 = 1e3
var_C = C_obs[band]
sig_C = np.sqrt(var_C)
obs = y[eidx]
_w = np.ones(nT - 1)
_w0 = np.ones(nT - 1) * 100
while not converged and itera < MAXITER:
"""
Robust smoothing...
using the method from the garcia
dct paper
"""
r = x[eidx] - y[eidx]
"""
h is the leverage from the
hat matrix ege the inverse matrix
"""
h = 0.0009
"""
Robust estimate of the standard
deviation of residuals
"""
#sig_rob = 1.4826 * np.median(r - np.median(r))
ui = r / (sig_C)
#Wr[eidx] = (1-(ui/4.685)**2)**2
# mask bad
#bad = np.abs(ui/4.685)>2
#Wr[eidx][bad]=0
"""
*- edge preserving --*
So this should now be fixed.
The functional is
w = T / sqrt( T**2 + (dx * alpha**2)**2 )
"""
"""
The derivative has to be scale by
alpha
"""
#dx = np.diff(x) * alpha**2
#ww[1:] = T/(np.sqrt(T**2 + dx**2))
dx = np.diff(x) * alpha**2
# only want drop in NIR
dx[dx > 0] = 0
"""
Thereshol T has to be scaled relative
to alpha (alpha actually works well)
"""
T = 0.7 * alpha
# calulcate the edge preserving functional w
_w = T / np.sqrt(T**2 + dx**2)
_w = np.exp(-dx**2 / T**2)
"""
Scaling from CERC
"""
wScale = nObs / np.sum(_w)
_w = _w * wScale
#_w = scipy.ndimage.minimum_filter(_w, 8)
co1 = np.sqrt(np.sum((x0[eidx] - obs)**2 / var_C) / nObs)
# faster version
cost.append(co1)
# remove nan
if itera > 2:
"""
Allow for some initial robust
outlier rejection
"""
np.place(_w, np.isnan(_w), 1)
W.data[0, :-1] = _w
else:
_w[:] = 1
"""
Remove edge effects
-- force every outside
in the edge windows back to 1
"""
W.data[0, :5] = 1
W.data[0, -5:] = 1
"""
I cant figure out the flat formulat to do below
"""
reg = alpha * bm.dot_mm(bm.dot_mm(D.T, W), D)
dl = reg.data[2][:-1]
di = reg.data[1] + Iobs * Wr
du = reg.data[0][1:]
ret = GTSV(dl, di, du, y, False, False, False, False)
x = ret[-2]
co = np.sum((x - x0)**2) / np.sum(x0**2) + 1e-6
cow = np.sum((_w - _w0)**2) / np.sum(_w0**2) + 1e-6
#if co < convT:
# converged=True
if cow < 1e5:
converged = True
if np.abs(co1 - co0) < 0.01:
converged = True
if itera < MINITER:
converged = False
if _w.min() < 0.1:
converged = True
x0 = x
_w0 = _w
co0 = co1
itera += 1
"""
Do some checking
So some times the algorithm freaks out
and over does the edge preserving...
Let's do some checks
1. Want one strong area with a small w
If w meets certain conditions optimise it a bit
- Sparsify it..
eg take the min and keep one min
place the min at the end of its window
"""
if np.any(_w < 0.3):
"""
sufficent edge process
to check whats going on
"""
"""
Check 1.
What percentage of timesteps are below
a threhsold --> eg is w all over the place?
and we've added to many edges
"""
reset = False
frac = (_w < 0.3).sum() / float(_w.shape[0])
# want no more than 5%
if frac > 0.1:
# re-set edge process
_w.fill(1)
reset = True
idx = np.nanargmin(_w)
val = _w[idx]
"""
Want to check a couple of things
1. If min w aligns with an observation
how far off the prediction are we?
--> this helps remove cases where
we have shot noise from clouds
or cloud shadows
"""
if eidx[idx]: # have an observ
if np.abs((y[idx] - x[idx])) / np.sqrt(C_obs[band]) > 2:
# it's a nonse break remove
_w.fill(1)
reset = True
else:
# check the surrounding pixels
# to find some obs to check
low = np.maximum(idx - 40, 0)
upp = np.minimum(idx + 40, 363)
mask = eidx[low:upp]
yy = y[low:upp][mask]
xx = x[low:upp][mask]
_ww = _w[low:upp][mask]
zz = np.abs((yy - xx)) / np.sqrt(C_obs[band])
cond = np.logical_and(_ww < 0.3, zz > 2).sum()
if cond > 4:
_w.fill(1)
reset = True
"""
Resolve it...
"""
if not reset:
"""
So this is probably a step
fix it up a bit
"""
#_w[:]=1
_w[idx] = 0.01
pass
W.data[0, :-1] = _w
"""
I cant figure out the flat formulat to do below
"""
reg = alpha * bm.dot_mm(bm.dot_mm(D.T, W), D)
dl = reg.data[2][:-1]
di = reg.data[1] + Iobs
du = reg.data[0][1:]
ret = GTSV(dl, di, du, y, False, False, False, False)
x = ret[-2]
else:
# use the smooth solution
x = x0
if type(W) != None and do_edges == False:
"""
Weights matrix supplied already...
use this for edges...
"""
# make W into a bandmat matrix
W = bm.diag(W)
reg = alpha * bm.dot_mm(bm.dot_mm(D.T, W), D)
dl = reg.data[2][:-1]
di = reg.data[1] + Iobs
du = reg.data[0][1:]
ret = GTSV(dl, di, du, y, False, False, False, False)
x = ret[-2]
"""
Add the MSE (eg obs unc estimate) to inflat unc...
"""
C = GTSV(dl, di, du, np.eye(nT), False, False, False, False)[-2]
unc = np.sqrt(C_obs[band] * np.diag(C))
return x, unc, W.data.flatten(), D