def test_band_of_inverse_from_chol(self, its=50): for it in range(its): size = random.choice([0, 1, randint(0, 10), randint(0, 100)]) chol_bm = gen_chol_factor_BandMat(size) depth = chol_bm.l + chol_bm.u band_of_inv_bm = bla.band_of_inverse_from_chol(chol_bm) assert not np.may_share_memory(band_of_inv_bm.data, chol_bm.data) mat_bm = (bm.dot_mm(chol_bm, chol_bm.T) if chol_bm.u == 0 else bm.dot_mm(chol_bm.T, chol_bm)) band_of_inv_full_good = fl.band_ec( depth, depth, np.eye(0, 0) if size == 0 else la.inv(mat_bm.full())) assert_allclose(band_of_inv_bm.full(), band_of_inv_full_good)
def test_band_of_inverse_from_chol(self, its=50): for it in range(its): size = random.choice([0, 1, randint(0, 10), randint(0, 100)]) chol_bm = gen_chol_factor_BandMat(size) depth = chol_bm.l + chol_bm.u band_of_inv_bm = bla.band_of_inverse_from_chol(chol_bm) assert not np.may_share_memory(band_of_inv_bm.data, chol_bm.data) mat_bm = (bm.dot_mm(chol_bm, chol_bm.T) if chol_bm.u == 0 else bm.dot_mm(chol_bm.T, chol_bm)) band_of_inv_full_good = fl.band_ec( depth, depth, np.eye(0, 0) if size == 0 else la.inv(mat_bm.full()) ) assert_allclose(band_of_inv_bm.full(), band_of_inv_full_good)
def prepare_mats(refl, alpha=1e3): """ Does some prep to produce matrices for the TDMA """ y = refl Iobs = y > 0 nT, ys, xs = y.shape # Create mats AC, BC, CC, DC = [ np.zeros([nT-1, ys, xs], dtype=np.float32), np.zeros([nT, ys, xs], dtype=np.float32), np.zeros([nT-1, ys, xs], dtype=np.float32), np.zeros([nT, ys, xs], dtype=np.float32)] # Create Diagonals matrices # for ease just use bandmat for this for now # Form diagonals I = np.eye(nT) #np.diag (np.ones(nx)) D = (I - np.roll(I, -1)).T #D = D.T.dot(D) D1A = np.zeros((nT, 2)) D1A[1:, 0] = np.diag(D, 1) D1A[:, 1] = np.diag(D, 0) # convert to banded matrices D = bm.BandMat(0, 1, D1A.T, transposed=False) D2 = bm.dot_mm(D.T, D) a = alpha * D2.data[2][:-1] b = alpha * D2.data[1] c = alpha * D2.data[0][1:] # add these AC[:]=a[:, None, None] BC[:]=b[:, None, None]+ Iobs CC[:]=c[:, None, None] DC[:]=y return AC, BC, CC, DC
def test__cholesky_banded(self, its=100): for it in range(its): size = random.choice([0, 1, randint(0, 10), randint(0, 100)]) if rand_bool(): mat_bm = gen_pos_def_BandMat(size, transposed=False) else: mat_bm = gen_symmetric_BandMat(size, transposed=False) # make it a bit more likely to be pos def bm.diag(mat_bm)[:] = np.abs(bm.diag(mat_bm)) + 0.1 depth = mat_bm.l lower = rand_bool() if lower: mat_half_data = mat_bm.data[depth:] else: mat_half_data = mat_bm.data[:(depth + 1)] overwrite = rand_bool() mat_half_data_arg = mat_half_data.copy() try: chol_data = bla._cholesky_banded( mat_half_data_arg, overwrite_ab=overwrite, lower=lower ) except la.LinAlgError as e: # First part of the message is e.g. "2-th leading minor". msgRe = (r'^' + re.escape(str(e)[:15]) + r'.*not positive definite$') with self.assertRaisesRegexp(la.LinAlgError, msgRe): sla.cholesky(mat_bm.full(), lower=lower) else: assert np.shape(chol_data) == (depth + 1, size) if lower: chol_bm = bm.BandMat(depth, 0, chol_data) mat_bm_again = bm.dot_mm(chol_bm, chol_bm.T) else: chol_bm = bm.BandMat(0, depth, chol_data) mat_bm_again = bm.dot_mm(chol_bm.T, chol_bm) assert_allclose(mat_bm_again.full(), mat_bm.full()) if size > 0: self.assertEqual( np.may_share_memory(chol_data, mat_half_data_arg), overwrite ) if not overwrite: assert np.all(mat_half_data_arg == mat_half_data)
def smooth(a, Phat, N, lambda_1, lambda_2): """ a: (N,) number of measurements at that timestep Phat: (N, 3) sum of measurements at that timestep N: num time steps lambda_1, lambda_2: regularization parameters solves the optimization problem (over P \in R^{Tx3}): minimize ||diag(a)*P-Phat||^2 + lambda_1/N*||D_2*P||^2 + lambda_2/N*||D_3*P||^2 returns: - P: (N, 3) matrix with full trajectory """ # A in Banded Matrix form A = bm.diag(1. * a) # D_2 and D_3 in Banded Matrix form transposed D_2_bm_T = bm.BandMat( 1, 1, np.hstack([ np.zeros((3, 1)), np.repeat([[1.], [-2.], [1.]], N - 2, axis=1), np.zeros((3, 1)) ])) D_3_bm_T = bm.BandMat( 2, 2, np.hstack([ np.zeros((5, 2)), np.repeat([[-1.], [2.], [0.], [-2.], [1.]], N - 4, axis=1), np.zeros((5, 2)) ])) # XP=B normal equations X = bm.dot_mm(A, A) + lambda_1 / N * bm.dot_mm( D_2_bm_T, D_2_bm_T.T) + lambda_2 / N * bm.dot_mm(D_3_bm_T, D_3_bm_T.T) l_and_u = (X.l, X.u) # lower and upper band bounds B = np.hstack([ np.expand_dims(bm.dot_mv(A, Phat[:, 0]), -1), np.expand_dims(bm.dot_mv(A, Phat[:, 1]), -1), np.expand_dims(bm.dot_mv(A, Phat[:, 2]), -1) ]) # solve normal equations P = solve_banded(l_and_u, X.data, B) return P
def test__cholesky_banded(self, its=100): for it in range(its): size = random.choice([0, 1, randint(0, 10), randint(0, 100)]) if rand_bool(): mat_bm = gen_pos_def_BandMat(size, transposed=False) else: mat_bm = gen_symmetric_BandMat(size, transposed=False) # make it a bit more likely to be pos def bm.diag(mat_bm)[:] = np.abs(bm.diag(mat_bm)) + 0.1 depth = mat_bm.l lower = rand_bool() if lower: mat_half_data = mat_bm.data[depth:] else: mat_half_data = mat_bm.data[:(depth + 1)] overwrite = rand_bool() mat_half_data_arg = mat_half_data.copy() try: chol_data = bla._cholesky_banded(mat_half_data_arg, overwrite_ab=overwrite, lower=lower) except la.LinAlgError as e: # First part of the message is e.g. "2-th leading minor". msgRe = (r'^' + re.escape(str(e)[:15]) + r'.*not positive definite$') with self.assertRaisesRegexp(la.LinAlgError, msgRe): sla.cholesky(mat_bm.full(), lower=lower) else: assert np.shape(chol_data) == (depth + 1, size) if lower: chol_bm = bm.BandMat(depth, 0, chol_data) mat_bm_again = bm.dot_mm(chol_bm, chol_bm.T) else: chol_bm = bm.BandMat(0, depth, chol_data) mat_bm_again = bm.dot_mm(chol_bm.T, chol_bm) assert_allclose(mat_bm_again.full(), mat_bm.full()) if size > 0: self.assertEqual( np.may_share_memory(chol_data, mat_half_data_arg), overwrite) if not overwrite: assert np.all(mat_half_data_arg == mat_half_data)
def test_cholesky(self, its=50): for it in range(its): size = random.choice([0, 1, randint(0, 10), randint(0, 100)]) mat_bm = gen_pos_def_BandMat(size) depth = mat_bm.l lower = rand_bool() alternative = rand_bool() chol_bm = bla.cholesky(mat_bm, lower=lower, alternative=alternative) assert chol_bm.l == (depth if lower else 0) assert chol_bm.u == (0 if lower else depth) assert not np.may_share_memory(chol_bm.data, mat_bm.data) if lower != alternative: mat_bm_again = bm.dot_mm(chol_bm, chol_bm.T) else: mat_bm_again = bm.dot_mm(chol_bm.T, chol_bm) assert_allclose(mat_bm_again.full(), mat_bm.full())
def test_dot_mm(self, its=50): for it in range(its): size = random.choice([0, 1, randint(0, 10), randint(0, 100)]) a_bm = gen_BandMat(size) b_bm = gen_BandMat(size) diag = None if rand_bool() else randn(size) diag_value = np.ones((size,)) if diag is None else diag a_full = a_bm.full() b_full = b_bm.full() c_bm = bm.dot_mm(a_bm, b_bm, diag=diag) c_full = np.dot(np.dot(a_full, np.diag(diag_value)), b_full) assert c_bm.l == a_bm.l + b_bm.l assert c_bm.u == a_bm.u + b_bm.u assert c_bm.size == size assert_allclose(c_bm.full(), c_full) assert not np.may_share_memory(c_bm.data, a_bm.data) assert not np.may_share_memory(c_bm.data, b_bm.data) if diag is not None: assert not np.may_share_memory(c_bm.data, diag)
def test_dot_mm(self, its=50): for it in range(its): size = random.choice([0, 1, randint(0, 10), randint(0, 100)]) a_bm = gen_BandMat(size) b_bm = gen_BandMat(size) diag = None if rand_bool() else randn(size) diag_value = np.ones((size, )) if diag is None else diag a_full = a_bm.full() b_full = b_bm.full() c_bm = bm.dot_mm(a_bm, b_bm, diag=diag) c_full = np.dot(np.dot(a_full, np.diag(diag_value)), b_full) assert c_bm.l == a_bm.l + b_bm.l assert c_bm.u == a_bm.u + b_bm.u assert c_bm.size == size assert_allclose(c_bm.full(), c_full) assert not np.may_share_memory(c_bm.data, a_bm.data) assert not np.may_share_memory(c_bm.data, b_bm.data) if diag is not None: assert not np.may_share_memory(c_bm.data, diag)
a_bm = bm.BandMat( 1, 1, np.array([ [0.0, 0.2, 0.3, 0.4, 0.5], [1.0, 0.9, 1.1, 0.8, 1.3], [0.3, 0.1, 0.5, 0.6, 0.0], ])) b_bm = bm.BandMat( 1, 0, np.array([ [1.0, 0.9, 1.1, 0.8, 1.3], [0.3, 0.1, 0.5, 0.6, 0.0], ])) c_bm = bm.dot_mm(a_bm.T, b_bm) d_bm = a_bm + b_bm a_full = a_bm.full() b_full = b_bm.full() c_full = c_bm.full() d_full = d_bm.full() print('a_full:') print(a_full) print() print('b_full:') print(b_full) print() print('np.dot(a_full.T, b_full):')
def solve_band(refl, solve_edge=True, W=None, alpha=1000, unc=False, drop=True): """ test run function """ Iobs = (refl>0) nObs = (Iobs).sum(axis=0) # scale alpha alpha = 1 alpha = alpha * np.nanmean(nObs) alpha = np.clip(alpha, 20, 250) if solve_edge: T = .6 * alpha nT, ys, xs = refl.shape # make D matrix I = np.eye(nT) #np.diag (np.ones(nx)) D = (I - np.roll(I, -1)).T #D = D.T.dot(D) D1A = np.zeros((nT, 2)) D1A[1:, 0] = np.diag(D, 1) D1A[:, 1] = np.diag(D, 0) # convert to banded matrices _D = bm.BandMat(0, 1, D1A.T, transposed=False) # create mats A, B, C, D = prepare_mats(refl, alpha=alpha) # Initial run with no smoothing X = TDMA_MAT(A, B, C, D) W0 = np.ones_like(X)*100 WW = np.ones_like(X) X0 = np.zeros_like(D) CONV = np.zeros((ys, xs)).astype(np.bool) Nits = np.zeros((ys, xs), dtype=np.int) for i in range(5): # calculate weights order_n = 1 #dx = np.diff(X, axis=0, n=order_n)* alpha**2 dx = np.gradient(X, axis=0) * alpha**2 # enforce direction constraint if drop != None: if drop: dx[dx>0]=0 else: dx[dx<=0]=0 # eg frst 30 days and last 30 days _w = np.exp(-dx**2 / T) """ Scaling from CERC """ y,x=30, 57 wScale = nObs / np.sum(_w, axis=0) _w = _w * wScale _w = np.clip(_w, 0., 1) np.place(_w, np.isnan(_w), 1) WW[:]=_w """ Update the regularisation matrix with the weights -- This involves a dot product with a diag eg D.T W D so i don't how to do this atm This is the slowest so if this can be done fast much better """ for y in range(ys): for x in range(xs): if not CONV[y,x]: # no convergence keep going n = alpha * bm.dot_mm(_D.T, _D, diag=WW[:, y, x].astype(np.float)) a = n.data[2][:-1] b = n.data[1] + Iobs[:, y, x] c = n.data[0][1:] # update matrices A[:, y, x]=a B[:, y, x]=b C[:, y, x]=c Nits[y,x]+=1 """ Run again with new weights """ X = TDMA_MAT(A, B, C, D) """ assess convergence -eg are the weights still changing """ cow = np.sum(np.abs((WW - W0) / W0), axis=0)+ 1e-6 CONV[cow<1e-2]=True MINITER=3 if i< MINITER: CONV[:]=False CONV[WW.min(axis=0)<0.1]=True W0 = WW X0 = X """ Want to check steps and re-inforce real steps """ W, keptEdges, Sch = refine_edges(WW, X, refl) """ Resolve with refined edges """ for y in range(ys): for x in range(xs): n = alpha * bm.dot_mm(_D.T, _D, diag=W[:, y, x].astype(np.float)) a = n.data[2][:-1] b = n.data[1] + Iobs[:, y, x] c = n.data[0][1:] # update matrices A[:, y, x]=a B[:, y, x]=b C[:, y, x]=c Nits[y,x]+=1 """ Run again with the refined weights """ #y,x=43, 62 if unc: X = TDMA_MAT(A, B, C, D) DD = np.copy(D).astype(np.float32) Inv = np.zeros_like(X) # only need the middle month tbh for t in range(20, nT-20): DD[:]=0 DD[t]=1 Inv[t]=TDMA_MAT(A, B, C, DD)[t] return X, Inv, W, CONV, Nits, Sch else: X = TDMA_MAT(A, B, C, D) return X, W, CONV, Nits, Sch else: """ Edge has been provided so use that """ Iobs = (refl>0) nObs = (Iobs).sum(axis=0) nT, ys, xs = refl.shape # make D matrix I = np.eye(nT) #np.diag (np.ones(nx)) D = (I - np.roll(I, -1)).T #D = D.T.dot(D) D1A = np.zeros((nT, 2)) D1A[1:, 0] = np.diag(D, 1) D1A[:, 1] = np.diag(D, 0) # convert to banded matrices _D = bm.BandMat(0, 1, D1A.T, transposed=False) # create mats A, B, C, D = prepare_mats(refl, alpha=alpha) # add the weights matrix for y in range(ys): for x in range(xs): n = alpha * bm.dot_mm(_D.T, _D, diag=W[:, y, x].astype(np.float)) a = n.data[2][:-1] b = n.data[1] + Iobs[:, y, x] c = n.data[0][1:] # update matrices A[:, y, x]=a B[:, y, x]=b C[:, y, x]=c """ Run again with the refined weights """ X = TDMA_MAT(A, B, C, D) return X, W
def unit_variance_mlpg_matrix(windows, T): """Compute MLPG matrix assuming input is normalized to have unit-variances. Let :math:`\mu` is the input mean sequence (``num_windows*T x static_dim``), :math:`W` is a window matrix ``(T x num_windows*T)``, assuming input is normalized to have unit-variances, MLPG can be written as follows: .. math:: y = R \mu where .. math:: R = (W^{T} W)^{-1} W^{T} Here we call :math:`R` as the MLPG matrix. Args: windows: (list): List of windows. T (int): Number of frames. Returns: numpy.ndarray: MLPG matrix (``T x nun_windows*T``). See also: :func:`nnmnkwii.autograd.UnitVarianceMLPG`, :func:`nnmnkwii.paramgen.mlpg`. Examples: >>> from nnmnkwii import paramgen as G >>> import numpy as np >>> windows = [ ... (0, 0, np.array([1.0])), ... (1, 1, np.array([-0.5, 0.0, 0.5])), ... (1, 1, np.array([1.0, -2.0, 1.0])), ... ] >>> G.unit_variance_mlpg_matrix(windows, 3) array([[ 2.73835927e-01, 1.95121944e-01, 9.20177400e-02, 9.75609720e-02, -9.09090936e-02, -9.75609720e-02, -3.52549881e-01, -2.43902430e-02, 1.10864742e-02], [ 1.95121944e-01, 3.41463417e-01, 1.95121944e-01, 1.70731708e-01, -5.55111512e-17, -1.70731708e-01, -4.87804860e-02, -2.92682916e-01, -4.87804860e-02], [ 9.20177400e-02, 1.95121944e-01, 2.73835927e-01, 9.75609720e-02, 9.09090936e-02, -9.75609720e-02, 1.10864742e-02, -2.43902430e-02, -3.52549881e-01]], dtype=float32) """ win_mats = build_win_mats(windows, T) sdw = np.max([win_mat.l + win_mat.u for win_mat in win_mats]) max_win_width = np.max([max(win_mat.l, win_mat.u) for win_mat in win_mats]) P = bm.zeros(sdw, sdw, T) # set edge precitions to zero precisions = bm.zeros(0, 0, T) precisions.data[:, max_win_width:-max_win_width] += 1.0 mod_win_mats = [] for win_index, win_mat in enumerate(win_mats): if win_index != 0: # use zero precisions for dynamic features mod_win_mat = bm.dot_mm(precisions, win_mat) bm.dot_mm_plus_equals(mod_win_mat.T, win_mat, target_bm=P) mod_win_mats.append(mod_win_mat) else: # static features bm.dot_mm_plus_equals(win_mat.T, win_mat, target_bm=P) mod_win_mats.append(win_mat) chol_bm = bla.cholesky(P, lower=True) Pinv = cholesky_inv_banded(chol_bm.full(), width=chol_bm.l + chol_bm.u + 1) cocatenated_window = full_window_mat(mod_win_mats, T) return Pinv.dot(cocatenated_window.T).astype(np.float32)
def mlpg_grad(mean_frames, variance_frames, windows, grad_output): """MLPG gradient computation Parameters are same as :func:`nnmnkwii.paramgen.mlpg` except for ``grad_output``. See the function docmenent for what the parameters mean. Let :math:`d` is the index of static features, :math:`l` is the index of windows, gradients :math:`g_{d,l}` can be computed by: .. math:: g_{d,l} = (\sum_{l} W_{l}^{T}P_{d,l}W_{l})^{-1} W_{l}^{T}P_{d,l} where :math:`W_{l}` is a banded window matrix and :math:`P_{d,l}` is a diagonal precision matrix. Assuming the variances are diagonals, MLPG can be performed in dimention-by-dimention efficiently. Let :math:`o_{d}` be ``T`` dimentional back-propagated gradients, the resulting gradients :math:`g'_{l,d}` to be propagated are computed as follows: .. math:: g'_{d,l} = o_{d}^{T} g_{d,l} Args: mean_frames (numpy.ndarray): Means. variance_frames (numpy.ndarray): Variances. windows (list): Windows. grad_output: Backpropagated output gradient, shape (``T x static_dim``) Returns: numpy.ndarray: Gradients to be back propagated, shape: (``T x D``) See also: :func:`nnmnkwii.autograd.mlpg`, :class:`nnmnkwii.autograd.MLPG` """ T, D = mean_frames.shape win_mats = build_win_mats(windows, T) static_dim = D // len(windows) max_win_width = np.max([max(win_mat.l, win_mat.u) for win_mat in win_mats]) grads = np.zeros((T, D), dtype=np.float32) for d in range(static_dim): sdw = max([win_mat.l + win_mat.u for win_mat in win_mats]) # R: \sum_{l} W_{l}^{T}P_{d,l}W_{l} R = bm.zeros(sdw, sdw, T) # overwritten in the loop # dtype = np.float64 for bandmat precisions = np.zeros((len(windows), T), dtype=np.float64) for win_idx, win_mat in enumerate(win_mats): precisions[win_idx] = 1 / \ variance_frames[:, win_idx * static_dim + d] # use zero precisions at edge frames for dynamic features if win_idx != 0: precisions[win_idx, :max_win_width] = 0 precisions[win_idx, -max_win_width:] = 0 bm.dot_mm_plus_equals(win_mat.T, win_mat, target_bm=R, diag=precisions[win_idx]) for win_idx, win_mat in enumerate(win_mats): # r: W_{l}^{T}P_{d,l} r = bm.dot_mm(win_mat.T, bm.diag(precisions[win_idx])) # grad_{d, l} = R^{-1r} grad = solve_banded((R.l, R.u), R.data, r.full()) assert grad.shape == (T, T) # Finally we get grad for a particular dimension grads[:, win_idx * static_dim + d] = grad_output[:, d].T.dot(grad) return grads
np.array([ [0.0, 0.2, 0.3, 0.4, 0.5], [1.0, 0.9, 1.1, 0.8, 1.3], [0.3, 0.1, 0.5, 0.6, 0.0], ]) ) b_bm = bm.BandMat( 1, 0, np.array([ [1.0, 0.9, 1.1, 0.8, 1.3], [0.3, 0.1, 0.5, 0.6, 0.0], ]) ) c_bm = bm.dot_mm(a_bm.T, b_bm) d_bm = a_bm + b_bm a_full = a_bm.full() b_full = b_bm.full() c_full = c_bm.full() d_full = d_bm.full() print('a_full:') print(a_full) print() print('b_full:') print(b_full) print() print('np.dot(a_full.T, b_full):')
def solve_banded_fast(y, alpha, band=1, do_edges=True, W=None, rmse=0.1, _D=None): """ Speed up to actually solve the thing really fast using scipy.linalg.solve_banded(L.T, y) """ # do some prep work nT = y.shape[0] q = np.zeros(nT) eidx = y > 0 q[eidx] = 1 Iobs = q # unc #Iobs /= C_obs[band] #y /= C_obs[band] nObs = eidx.sum() """ *-- Alpha value --* This needs to be normalised by the number of observations OR DO I ?? """ """ make D matrix in flattened form faster to pre-compute as is it's a standard length """ #_D = None if _D == None: D = make_D_matrix(nT) else: D = _D """ get a smooth solution """ # weighting for outliers Wr = np.ones(nT) scale = (nObs) # re-scale alpha """ re-scale alpha So trying something a little different... Basically allow this to vary across the time-series too based on the number of samples in a local window. I think this should improve the scaling of alpha so we can keep edges where there are lots of samples.... """ #winN = np.ones(20) #nScale = np.convolve(eidx, winN, mode='same') #alpha = 0.5*alpha * nScale + 0.5*alpha * scale alpha = alpha * scale alpha = np.minimum(alpha, 200) alpha = np.maximum(alpha, 70) D2 = bm.dot_mm(D.T, D) dl = alpha * D2.data[2][:-1] di = alpha * D2.data[1] + Iobs * Wr du = alpha * D2.data[0][1:] ret = GTSV(dl, di, du, y, False, False, False, False) x0 = ret[-2] #import pdb; pdb.set_trace() """ *-- now do edge preserving --* """ cost = [] tx = np.arange(len(y))[eidx] if do_edges: # Define some stuff converged = False convT = 1e-5 itera = 0 MAXITER = 10 MINITER = 8 T = 0.01 # threshold in the change needed... # Define the weights matrix _wg = np.ones(nT) W = bm.diag(_wg) # run until convergence x = x0 R = [] co0 = 1e3 var_C = C_obs[band] sig_C = np.sqrt(var_C) obs = y[eidx] _w = np.ones(nT - 1) _w0 = np.ones(nT - 1) * 100 while not converged and itera < MAXITER: """ Robust smoothing... using the method from the garcia dct paper """ r = x[eidx] - y[eidx] """ h is the leverage from the hat matrix ege the inverse matrix """ h = 0.0009 """ Robust estimate of the standard deviation of residuals """ #sig_rob = 1.4826 * np.median(r - np.median(r)) ui = r / (sig_C) #Wr[eidx] = (1-(ui/4.685)**2)**2 # mask bad #bad = np.abs(ui/4.685)>2 #Wr[eidx][bad]=0 """ *- edge preserving --* So this should now be fixed. The functional is w = T / sqrt( T**2 + (dx * alpha**2)**2 ) """ """ The derivative has to be scale by alpha """ #dx = np.diff(x) * alpha**2 #ww[1:] = T/(np.sqrt(T**2 + dx**2)) dx = np.diff(x) * alpha**2 # only want drop in NIR dx[dx > 0] = 0 """ Thereshol T has to be scaled relative to alpha (alpha actually works well) """ T = 0.7 * alpha # calulcate the edge preserving functional w _w = T / np.sqrt(T**2 + dx**2) _w = np.exp(-dx**2 / T**2) """ Scaling from CERC """ wScale = nObs / np.sum(_w) _w = _w * wScale #_w = scipy.ndimage.minimum_filter(_w, 8) co1 = np.sqrt(np.sum((x0[eidx] - obs)**2 / var_C) / nObs) # faster version cost.append(co1) # remove nan if itera > 2: """ Allow for some initial robust outlier rejection """ np.place(_w, np.isnan(_w), 1) W.data[0, :-1] = _w else: _w[:] = 1 """ Remove edge effects -- force every outside in the edge windows back to 1 """ W.data[0, :5] = 1 W.data[0, -5:] = 1 """ I cant figure out the flat formulat to do below """ reg = alpha * bm.dot_mm(bm.dot_mm(D.T, W), D) dl = reg.data[2][:-1] di = reg.data[1] + Iobs * Wr du = reg.data[0][1:] ret = GTSV(dl, di, du, y, False, False, False, False) x = ret[-2] co = np.sum((x - x0)**2) / np.sum(x0**2) + 1e-6 cow = np.sum((_w - _w0)**2) / np.sum(_w0**2) + 1e-6 #if co < convT: # converged=True if cow < 1e5: converged = True if np.abs(co1 - co0) < 0.01: converged = True if itera < MINITER: converged = False if _w.min() < 0.1: converged = True x0 = x _w0 = _w co0 = co1 itera += 1 """ Do some checking So some times the algorithm freaks out and over does the edge preserving... Let's do some checks 1. Want one strong area with a small w If w meets certain conditions optimise it a bit - Sparsify it.. eg take the min and keep one min place the min at the end of its window """ if np.any(_w < 0.3): """ sufficent edge process to check whats going on """ """ Check 1. What percentage of timesteps are below a threhsold --> eg is w all over the place? and we've added to many edges """ reset = False frac = (_w < 0.3).sum() / float(_w.shape[0]) # want no more than 5% if frac > 0.1: # re-set edge process _w.fill(1) reset = True idx = np.nanargmin(_w) val = _w[idx] """ Want to check a couple of things 1. If min w aligns with an observation how far off the prediction are we? --> this helps remove cases where we have shot noise from clouds or cloud shadows """ if eidx[idx]: # have an observ if np.abs((y[idx] - x[idx])) / np.sqrt(C_obs[band]) > 2: # it's a nonse break remove _w.fill(1) reset = True else: # check the surrounding pixels # to find some obs to check low = np.maximum(idx - 40, 0) upp = np.minimum(idx + 40, 363) mask = eidx[low:upp] yy = y[low:upp][mask] xx = x[low:upp][mask] _ww = _w[low:upp][mask] zz = np.abs((yy - xx)) / np.sqrt(C_obs[band]) cond = np.logical_and(_ww < 0.3, zz > 2).sum() if cond > 4: _w.fill(1) reset = True """ Resolve it... """ if not reset: """ So this is probably a step fix it up a bit """ #_w[:]=1 _w[idx] = 0.01 pass W.data[0, :-1] = _w """ I cant figure out the flat formulat to do below """ reg = alpha * bm.dot_mm(bm.dot_mm(D.T, W), D) dl = reg.data[2][:-1] di = reg.data[1] + Iobs du = reg.data[0][1:] ret = GTSV(dl, di, du, y, False, False, False, False) x = ret[-2] else: # use the smooth solution x = x0 if type(W) != None and do_edges == False: """ Weights matrix supplied already... use this for edges... """ # make W into a bandmat matrix W = bm.diag(W) reg = alpha * bm.dot_mm(bm.dot_mm(D.T, W), D) dl = reg.data[2][:-1] di = reg.data[1] + Iobs du = reg.data[0][1:] ret = GTSV(dl, di, du, y, False, False, False, False) x = ret[-2] """ Add the MSE (eg obs unc estimate) to inflat unc... """ C = GTSV(dl, di, du, np.eye(nT), False, False, False, False)[-2] unc = np.sqrt(C_obs[band] * np.diag(C)) return x, unc, W.data.flatten(), D