def Relevance_Vector_Classification_Training(X,Y,kernel_mode): A=np.zeros((X.shape[0]+1,X.shape[0]+1),float) np.fill_diagonal(A,1e-5) sigmoids=[sigmoid_function(y)*(1-sigmoid_function(y)) for y in Y] B=np.diag(sigmoids) designMatrix = tuning.design_matrix(X.shape[0],kernel_mode,X) num_iter=0 while(True): weightMaxPosteriori, Sigma = probability_estimators.second_order_statistics(designMatrix) gamas = np.ones(X.shape[0]+1, float) - np.diag(A)*np.diag(Sigma) deleted_indexes = [] while (True): A_old = np.copy(A) for j in range(1, A.shape[0]): A[j, j] = gamas[j] / (weightMaxPosteriori[j] ** 2) if (A[j, j] > 10e8): deleted_indexes.append(j) if (len(deleted_indexes) > 0): A = np.delete(A, deleted_indexes, 0) A = np.delete(A, deleted_indexes, 1) A_old = np.delete(A_old, deleted_indexes, 0) A_old = np.delete(A_old, deleted_indexes, 1) deleted_indexes[:] = [x - 1 for x in deleted_indexes] X = np.delete(X, deleted_indexes, 0) Y = np.delete(Y, deleted_indexes, 0) B = np.delete(B, deleted_indexes, 0) B = np.delete(B, deleted_indexes, 1) designMatrix = tuning.design_matrix(X.shape[0], kernel_mode, X) deleted_indexes.clear() # Convergence criterion if (np.abs(np.trace(A) - np.trace(A_old)))< 1e-3 or num_iter>20000: break num_iter+=1 weightMaxPosteriori, _ = probability_estimators.second_order_statistics(designMatrix, A, B, Y) return X, weightMaxPosteriori
def test(): X = np.linspace(-10, 10, 100) Y = np.sin(np.abs(X)) / np.abs(X) # plt.plot(Y,'r') variance = 10**(-2) A = np.zeros((X.shape[0] + 1, X.shape[0] + 1), float) B = np.zeros((X.shape[0], X.shape[0]), float) np.fill_diagonal(A, 1) np.fill_diagonal(B, (1 / variance)) designMatrix = tuning.design_matrix(X.shape[0], "linear_spline", X, Y) mean, Sigma = probability_estimators.second_order_statistics( designMatrix, A, B, Y) gamas = np.ones(X.shape[0] + 1, float) - np.diag(A) * np.diag(Sigma) max_iter = 0 deleted_indexes = [] while (True): A_old = np.copy(A) for j in range(1, A.shape[0]): A[j, j] = gamas[j] / (mean[j]**2) if (A[j, j] > 1000): deleted_indexes.append(j) if (len(deleted_indexes) > 0): debug = 0 A = np.delete(A, deleted_indexes, 0) A = np.delete(A, deleted_indexes, 1) A_old = np.delete(A_old, deleted_indexes, 0) A_old = np.delete(A_old, deleted_indexes, 1) deleted_indexes[:] = [x - 1 for x in deleted_indexes] B = np.delete(B, deleted_indexes, 0) B = np.delete(B, deleted_indexes, 1) X = np.delete(Y, deleted_indexes, 0) Y = np.delete(Y, deleted_indexes, 0) deleted_indexes.clear() debug = 0 # Convergence criterion if (np.abs(np.trace(A) - np.trace(A_old))) < 0 and max_iter > 1: break max_iter += 1 designMatrix = tuning.design_matrix(X.shape[0], "linear_spline", X, Y) mean, Sigma = probability_estimators.second_order_statistics( designMatrix, A, B, Y) gamas = np.ones(X.shape[0] + 1, float) - np.diag(A) * np.diag(Sigma) res = np.diag(A) debug = 0
def train(kernel_mode): X = np.linspace(-10, 10, 100) Y = np.sin(np.abs(X)) / np.abs(X) # plt.plot(Y,'r') variance = 0.0001 A = np.zeros((X.shape[0] + 1, X.shape[0] + 1), float) B = np.zeros((X.shape[0], X.shape[0]), float) np.fill_diagonal(A, 1e-5) # A = A * np.random.normal(0, 6, A.shape[0]) np.fill_diagonal(B, (1 / variance)) designMatrix = tuning.design_matrix(X.shape[0], kernel_mode, X) max_iter = 0 deleted_indexes = [] while (True): A_old = np.copy(A) mean, Sigma = probability_estimators.second_order_statistics( designMatrix, A, B, Y) gamas = np.ones(X.shape[0] + 1, float) - np.diag(A) * np.diag(Sigma) # gamas = 1 - np.diag(A) * np.diag(Sigma) for j in range(0, A.shape[0]): A[j, j] = gamas[j] / (mean[j]**2) if (A[j, j] > 1e9 and j > 0): deleted_indexes.append(j) # B = np.zeros((X.shape[0], X.shape[0]), float) # np.fill_diagonal(B, (1 / tuning.common_noise_variance(Y, designMatrix, mean, Sigma, gamas))) if (len(deleted_indexes) > 0): A = np.delete(A, deleted_indexes, 0) A = np.delete(A, deleted_indexes, 1) A_old = np.delete(A_old, deleted_indexes, 0) A_old = np.delete(A_old, deleted_indexes, 1) deleted_indexes[:] = [x - 1 for x in deleted_indexes] X = np.delete(X, deleted_indexes, 0) Y = np.delete(Y, deleted_indexes, 0) B = np.delete(B, deleted_indexes, 0) B = np.delete(B, deleted_indexes, 1) designMatrix = tuning.design_matrix(X.shape[0], kernel_mode, X) deleted_indexes.clear() debug = 0 # Convergence criterion if (np.abs(np.trace(A) - np.trace(A_old))) < 1e-3: break if max_iter > 400: break max_iter += 1 print(max_iter) mean, _ = probability_estimators.second_order_statistics( designMatrix, A, B, Y) res = np.diag(A) debug = 0 return X, mean
def test(): X = np.linspace(-10, 10, 100) Y = np.sin(np.abs(X)) / np.abs(X) # plt.plot(Y,'r') variance = 0.01 A = np.zeros((X.shape[0] + 1, X.shape[0] + 1), float) B = np.zeros((X.shape[0], X.shape[0]), float) np.fill_diagonal(A, 1) np.fill_diagonal(B, (1 / variance)) designMatrix = tuning.design_matrix(X.shape[0], "linear_spline", X) mean, Sigma = probability_estimators.second_order_statistics(designMatrix, A, B, Y) gamas = np.ones(X.shape[0] + 1, float) - np.diag(A) * np.diag(Sigma) max_iter = 0 deleted_indexes = [] while (True): A_old = np.copy(A) for j in range(1, A.shape[0]): A[j, j] = gamas[j] / (mean[j] ** 2) if (A[j, j] > 1e9): deleted_indexes.append(j) debug = 0 if (len(deleted_indexes) > 0): debug = 0 A = np.delete(A, deleted_indexes, 0) A = np.delete(A, deleted_indexes, 1) A_old = np.delete(A_old, deleted_indexes, 0) A_old = np.delete(A_old, deleted_indexes, 1) deleted_indexes[:] = [x - 1 for x in deleted_indexes] B = np.delete(B, deleted_indexes, 0) B = np.delete(B, deleted_indexes, 1) X = np.delete(Y, deleted_indexes, 0) Y = np.delete(Y, deleted_indexes, 0) deleted_indexes.clear() debug = 0 # Covergence criterion suggested from RVM+Explained # which can be found on the Literature folder if (np.abs(np.trace(A) - np.trace(A_old))) and max_iter > 1: break max_iter += 1 designMatrix = tuning.design_matrix(X.shape[0], "linear_spline", X) mean, Sigma = probability_estimators.second_order_statistics(designMatrix, A, B, Y) gamas = np.ones(X.shape[0] + 1, float) - np.diag(A) * np.diag(Sigma) res = np.diag(A) res2 = np.diag(A_old) debug = 0 X_new = np.random.uniform(-10, 10, 100) Y_true = np.sin(np.abs(X_new)) / np.abs(X_new) Y_new = [] MSE=[] for i in range(0, 100): y = 0 for j in range(1, 5): Y_new.append(np.sum(y) + res[0]) MSE.append(np.sqrt(Y_true[i] - Y_new[i])) plt.plot(X_new, Y_new) plt.show() debug=0
def iterative_reweighted_least_squares_algorithm2(X, target, kernel_mode="rbf", max_iter=25, alpha_threshold=10e8, gradient_threshold=10e-6, overshoot_criterion=power( 2, -8)): weights = np.zeros(X.shape[0] + 1) A = np.zeros((X.shape[0] + 1, X.shape[0] + 1), float) np.fill_diagonal(A, 1e-5) designMatrix = tuning.design_matrix(X.shape[0], kernel_mode, X) deleted_indexes = [] for it in range(1000): A_old = np.copy(A) for j in range(1, A.shape[0]): if it > 0: A[j, j] = gamas[j] / (weightMaxPosteriori[j]**2) if (A[j, j] > 10e8): deleted_indexes.append(j) if (len(deleted_indexes) > 0): A = np.delete(A, deleted_indexes, 0) A = np.delete(A, deleted_indexes, 1) A_old = np.delete(A_old, deleted_indexes, 0) A_old = np.delete(A_old, deleted_indexes, 1) deleted_indexes[:] = [x - 1 for x in deleted_indexes] X = np.delete(X, deleted_indexes, 0) target = np.delete(target, deleted_indexes, 0) B = np.delete(B, deleted_indexes, 0) designMatrix = tuning.design_matrix(X.shape[0], "rbf", X) weights = np.delete(X, deleted_indexes, 0) deleted_indexes.clear() debug = 0 y = np.dot(designMatrix, weights) y = np.asarray([almost_sigmoid(y[i]) for i in range(y.shape[0])]) # Negative log function minimization data_term = 0 for i in range(target.shape[0]): if target[i] == 1: data_term -= math.log(y[i]) else: data_term -= math.log(1.0 - y[i]) regulariser = -0.5 * np.sum(np.dot(weights.T, np.dot(A, weights))) error = (data_term + regulariser / 2.0) / weights.shape[0] # tempB = [y[i] * (1 - y[i]) for i in range(len(y))] # B = np.asarray(tempB) for g in range(max_iter): temp_weight = [y[k] * (1 - y[k]) for k in range(y.shape[0])] irls_weights = np.diagflat(temp_weight) Hessian = np.dot(designMatrix.T, np.dot(irls_weights, designMatrix)) Hessian += np.diag(A) Sigma = np.linalg.inv(Hessian) e = np.asarray([target[k] - y[k] for k in range(y.shape[0])]) gradient_error = np.dot(designMatrix.T, e) - np.dot(A, weights) gradient_error -= A * weights delta_weights = np.dot(Sigma, gradient_error) if (g >= 2 and np.linalg.norm(gradient_error) / weights.shape[0] < gradient_threshold): break delta_weights = np.dot(Sigma, gradient_error) l = 1 # Overshooting part while (l > overshoot_criterion): weightMaxPosteriori = weights + l * delta_weights y = np.dot(designMatrix, weightMaxPosteriori) y = np.asarray( [almost_sigmoid(y[i]) for i in range(y.shape[0])]) data_term = 0 for i in range(target.shape[0]): if target[i] == 1: try: data_term -= math.log(y[i]) except ValueError: lo = y[i] debug = 0 else: data_term -= math.log(1.0 - y[i]) regulariser = -0.5 * np.dot(weights.T, np.dot(A, weights)) error_update = (data_term + regulariser / 2.0) / weights.shape[0] if (error_update > error): l /= 2.0 else: weights = weightMaxPosteriori break weights = weightMaxPosteriori gamas = np.ones(X.shape[0], float) - np.diag(A) * np.diag(Sigma) A_old = np.copy(A) deleted_indexes = [] for j in range(1, A.shape[0]): A[j, j] = gamas[j] / (weightMaxPosteriori[j]**2) if (A[j, j] > 10e8): deleted_indexes.append(j) if (len(deleted_indexes) > 0): A = np.delete(A, deleted_indexes, 0) A = np.delete(A, deleted_indexes, 1) A_old = np.delete(A_old, deleted_indexes, 0) A_old = np.delete(A_old, deleted_indexes, 1) deleted_indexes[:] = [x - 1 for x in deleted_indexes] X = np.delete(X, deleted_indexes, 0) target = np.delete(target, deleted_indexes, 0) B = np.delete(B, deleted_indexes, 0) # B = np.delete(B, deleted_indexes, 1) designMatrix = tuning.design_matrix(X.shape[0], "rbf", X) weights = np.delete(X, deleted_indexes, 0) deleted_indexes.clear() if (np.abs(np.trace(A) - np.trace(A_old))) < 1e-3: break return weights, X