def obj_func(self, X, W, n_splits=3): """ error :param X: :param W: :param n_splits: :return: """ # ## TODO: train learner and calculate crossover Root Mean Square Error # skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=233) kf = KFold(n_splits=n_splits) error = 0 kl = 0. desired_degree = np.array([ self.sparse_degree, ] * self.n_hidden) for train_index, test_index in kf.split(self.X): X_train, X_test = X[train_index], X[test_index] y_train, y_test = self.T[train_index], self.T[test_index] H = expit(np.dot(X_train, W)) B = np.dot(linalg.pinv(H), y_train) H_ = expit(np.dot(X_test, W)) output = np.dot(H_, B) error += np.sqrt(mean_squared_error(y_test, output)) kl_split = 0. real_degree = H.mean(axis=0) # n_hidden dim for p1, p2 in zip(real_degree, desired_degree): entr = entropy([p1, 1 - p1], [p2, 1 - p2]) kl_split += entr kl += kl_split return kl / n_splits, error / n_splits
def obj_func_MSE(self, X, W, n_splits=3): """ error :param X: :param W: :param n_splits: :return: """ # ## TODO: train learner and calculate error of cross-over validation skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=233) error = 0 kl = 0. desired_degree = np.array([ self.sparse_degree, ] * self.n_hidden) for train_index, test_index in skf.split(self.X, self.T.argmax(axis=1)): X_train, X_test = X[train_index], X[test_index] y_train, y_test = self.T[train_index], self.T[test_index] H = expit(np.dot(X_train, W)) B = np.dot(linalg.pinv(H), y_train) H_ = expit(np.dot(X_test, W)) output = np.dot(H_, B) # error += (1 - accuracy_score(y_test.argmax(axis=1), output.argmax(axis=1))) error += mean_squared_error(y_test, output) kl_split = 0. real_degree = H.mean(axis=0) # n_hidden dim for p1, p2 in zip(real_degree, desired_degree): entr = entropy([p1, 1 - p1], [p2, 1 - p2]) kl_split += entr kl += kl_split return kl / n_splits, error / n_splits
def obj_func_v2(self, X, W, n_splits=3): """ error :param X: :param W: :param n_splits: :return: """ # ## TODO: train learner and calculate crossover Root Mean Square Error # skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=233) kf = KFold(n_splits=n_splits) error = 0 kl = 0. desired_degree = np.array([ self.sparse_degree, ] * self.n_hidden) for train_index, test_index in kf.split(self.X): X_train, X_test = X[train_index], X[test_index] y_train, y_test = self.T[train_index], self.T[test_index] H = expit(np.dot(X_train, W)) B = np.dot(linalg.pinv(H), y_train) H_ = expit(np.dot(X_test, W)) output = np.dot(H_, B) error += np.sqrt(mean_squared_error(y_test, output)) S = np.linalg.norm( W, ord=1 ) #np.linalg.norm(np.dot(X_test[:, :-1], B.transpose()), ord=2)/np.linalg.norm(np.dot(X_test[:, :-1], B.transpose()), ord=1) kl += S return kl / n_splits, error / n_splits
def predict(self, X, W=None): X_ = np.append(X, np.ones((X.shape[0], 1)), axis=1) if W is None: H = expit(np.dot(X_, self.best_W)) else: H = expit(np.dot(X_, W)) return H
def _predict_proba_lr(self, X): prob = np.ravel(self.decision_function(X)) expit(prob, out=prob) if prob.ndim == 1: return np.vstack([1 - prob, prob]).T else: # OvR normalization, like LibLinear's predict_probability prob /= prob.sum(axis=1).reshape((prob.shape[0], -1)) return prob
def __fitness_func(self, vars): # cost function or optimized function skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=233) error = 0 X_ = np.append(self.X, np.ones((self.X.shape[0], 1)), axis=1) W = np.asarray(vars).reshape((self.X.shape[1] + 1, self.n_hidden)) for train_index, test_index in skf.split(X_, self.y): X_train, X_test = X_[train_index], X_[test_index] y_train, y_test = self.y_bin[train_index], self.y_bin[test_index] H = expit(np.dot(X_train, W)) B = np.dot(linalg.pinv(H), y_train) H_ = expit(np.dot(X_test, W)) output = np.dot(H_, B) error += ( 1. - accuracy_score(y_test.argmax(axis=1), output.argmax(axis=1))) # error += mean_squared_error(y_test, output) return error / 3.
def predict(self, X): XX = copy.deepcopy(X) H = None for i in range(len(self.W)): X_ = np.append(XX, np.ones((XX.shape[0], 1)), axis=1) H = expit(np.dot(X_, self.W[i])) XX = copy.deepcopy(H) output = np.dot(H, self.B) return output.argmax(axis=1)
def best_predict(self, X): index = self.get_best_W_index() W = self.W[index] B = self.B[index] X = copy.deepcopy(X) X_test = np.append(X, np.ones((X.shape[0], 1)), axis=1) H = expit(np.dot(X_test, W)) output = np.dot(H, B) return output.argmax(axis=1)
def binary_transition_smooth(x: Union[float, np.ndarray], xthr: float, S: float = 5.0) -> Union[float, np.ndarray]: """ Makes a nice transition from 1 to 0 as x increases (sort of inverse sigmoid) :param x: value to map (or array) :param xthr: threshold value at which output is 0.5 :param S: Shape factor :return: mapped value (or array) """ return 1 - expit(x / xthr * S - S)
def streaming_lr(t, x_t, y_t, betas, alpha, mu, invscaling=False): if betas is None: if len(x_t.shape) > 0: betas = np.zeros(x_t.shape[0]) else: betas = np.array(0.) p = expit(betas.dot(x_t)) alpha = alpha / pow(t, invscaling) if invscaling else alpha betas *= (1 - 2. * alpha * mu) betas += alpha * (y_t - p) * x_t return betas, alpha
def stick_breaking(Psi): """ Calculates the stickbreaking construction of the gaussian variables Psi Parameters ---------- Psi: [M x K-1] Gaussian Variables used for the stick breaking Returns ------- Pi: [M x K] Probability matrix using the logistic function and stick breaking """ Pi = np.zeros((np.shape(Psi)[0], np.shape(Psi)[1] + 1)) Pi[:, 0] = expit(Psi[:, 0]) Pi[:, 1:-1] = expit(Psi[:, 1:]) * np.cumprod(1 - expit(Psi[:, 0:-1]), axis=1) Pi[:, -1] = 1 - np.sum(Pi[:, 0:-1], axis=1) # Check for numerical instability if np.any(Pi[:, -1] < 0): Pi[Pi[:, -1] < 0, -1] = 0 # Set last weight to 0 Pi /= np.sum(Pi, axis=1)[:, None] # Normalize last weight to 0 return Pi
def obj_func(self, X, W, n_splits=3): """ error :param X: :param W: :param n_splits: :return: """ # ## TODO: train learner and calculate crossover Root Mean Square Error kf = KFold(n_splits=n_splits) error = 0. kl = 0. for train_index, test_index in kf.split(self.X): X_train, X_test = X[train_index], X[test_index] y_train, y_test = self.T[train_index], self.T[test_index] H = expit(np.dot(X_train, W)) B = np.dot(linalg.pinv(H), y_train) H_ = expit(np.dot(X_test, W)) output = np.dot(H_, B) error += np.sqrt(mean_squared_error(y_test, output)) kl += np.linalg.norm(H, ord=1) return kl/n_splits, error/n_splits
def lr_predict(x, weights, i, learnrate, regulizer, default_value=None): """ Predict value using logistic regression :param t: :param x: :param param: :return: """ if default_value is None: default_value = lambda: None value = weights.get("intercept", default_value()) for key in x: value += weights.get(key, default_value()) * x[key] value *= (1. - 2. * learnrate * regulizer) ** i return expit(value)
def __get_info(self, X_test, W, B, y_test=None): H = expit(np.dot(X_test, W)) y_ = np.dot(H, B).argmax(axis=1) # # calculate accuracy acc = None if y_test is not None: acc = accuracy_score(y_test, y_) # # calculate KL divergence kl = 0. desired_degree = np.array([self.sparse_degree, ] * self.n_hidden) real_degree = H.mean(axis=0) # n_hidden dim avg_activation = real_degree.mean() for p1, p2 in zip(real_degree, desired_degree): entr = entropy([p1, 1 - p1], [p2, 1 - p2]) kl += entr return y_, acc, kl, avg_activation
def fit(self, X, y): X, y, = copy.deepcopy(X), copy.deepcopy(y) self.y = y y_bin = y self.X, self.y_bin = X, y # # start evolving in MOEA num_variables = (self.X.shape[1] + 1) * self.n_hidden algorithm = NSGAII(Objectives(num_variables, 2, self.X, y_bin, self.n_hidden, sparse_degree=self.sparse_degree), population_size=self.n_pop) # MOEAD(Objectives(num_variables, 2, self.X, y_bin, self.n_hidden, sparse_degree=self.sparse_degree), # population_size=self.n_pop, neighborhood_size=int(self.n_pop/10)) # delta=0.5, eta=0.8 algorithm.run(self.max_iter) self.evo_result = algorithm.result print('total solution:', algorithm.result.__len__()) nondom_result = nondominated(algorithm.result) print('nondominated solution:', nondom_result.__len__()) self.nondom_solution = nondom_result self.W = [] self.B = [] for i in range(nondom_result.__len__()): s = nondom_result[i] W = np.asarray(s.variables).reshape(self.X.shape[1] + 1, self.n_hidden) X_ = np.append(self.X, np.ones((self.X.shape[0], 1)), axis=1) H = expit(np.dot(X_, W)) B = np.dot(linalg.pinv(H), y_bin) self.W.append(W) self.B.append(B) real_degree = H.mean(axis=0) # n_hidden dim avg_activation = real_degree.mean() print('NO.', i, ' obj:', s.objectives, 'AVG activation:', avg_activation) self.W = np.asarray(self.W) self.B = np.asarray(self.B) # # best W/B best_index = self.get_best_index() self.best_W = self.W[best_index] self.best_B = self.B[best_index] return self
def fit(self, X, y): X, y, = copy.deepcopy(X), copy.deepcopy(y) self.y = y y_bin = self.one2array(y, np.unique(y).shape[0]) self.classes_ = np.arange(y_bin.shape[1]) self.n_classes_ = self.classes_.__len__() self.X, self.y_bin = X, y_bin # # start evolving in MOEA num_variables = (self.X.shape[1] + 1) * self.n_hidden algorithm = MOEAD(Objectives(num_variables, 2, self.X, y_bin, self.n_hidden, sparse_degree=self.sparse_degree), population_size=self.n_pop, neighborhood_size=5) algorithm.run(self.max_iter) self.evo_result = algorithm.result print('total solution:', algorithm.result.__len__()) result = nondominated(algorithm.result) print('nondominated solution:', result.__len__()) self.solution = result self.W = [] self.B = [] self.voting_weight = [] for s in result: W = np.asarray(s.variables).reshape(self.X.shape[1] + 1, self.n_hidden) X_ = np.append(self.X, np.ones((self.X.shape[0], 1)), axis=1) H = expit(np.dot(X_, W)) B = np.dot(linalg.pinv(H), y_bin) voting_w_ = 1. / (s.objectives[0] + 10e-5) * self.mu + 1. / ( s.objectives[1] + 10e-5) * (1 - self.mu) self.voting_weight.append(voting_w_) self.W.append(W) self.B.append(B) self.voting_weight = np.asarray(self.voting_weight) self.W = np.asarray(self.W) self.B = np.asarray(self.B) return self
def fit(self, X, y): X, y, = copy.deepcopy(X), copy.deepcopy(y) self.y = y y_bin = self.one2array(y, np.unique(y).shape[0]) self.classes_ = np.arange(y_bin.shape[1]) self.n_classes_ = self.classes_.__len__() self.X, self.y_bin = X, y_bin bounds = np.array([ [-1., 1.], ] * (self.X.shape[1] + 1) * self.n_hidden) de = DiffEvolOptimizer(self.__fitness_func, bounds, self.n_pop, F=0.7, C=0.5) solution, fitness = de.optimize(ngen=self.max_iter) self.W = np.asarray(solution).reshape( (self.X.shape[1] + 1, self.n_hidden)) X_ = np.append(self.X, np.ones((self.X.shape[0], 1)), axis=1) H = expit(np.dot(X_, self.W)) self.B = np.dot(linalg.pinv(H), y_bin) return self
def predict(self, X): X_ = np.append(X, np.ones((X.shape[0], 1)), axis=1) output = expit(np.dot(X_, self.orth_W)) return output
def test_zero(self): self.assertAlmostEqual(expit(0.), 0.5, places=5)
def test_minus_inf(self): self.assertAlmostEqual(expit(-100), 0., places=5)
from sklearntools.validation import plot_tolerance, plot_roc,\ calibration_bin_plot, plot_roc_auc_for_bins if __name__ == '__main__': from sklearntools.validation import plot_curve_auc, roc_curve from sklearn.linear_model.logistic import LogisticRegression from scipy.special._ufuncs import expit import numpy as np from matplotlib import pyplot np.random.seed(1) m = 1000 n = 5 X = np.random.normal(size=(m, n)) beta = np.random.normal(size=n) y = np.random.binomial(n=1, p=expit(np.dot(X, beta))) model = LogisticRegression().fit(X, y) pred = model.predict_proba(X)[:, 1] pyplot.figure() plot_roc(y, pred, name='test_model') pyplot.savefig('test_roc_plot.png') pyplot.figure() plot_tolerance(y, pred, name='test_model', normalize=True) pyplot.savefig('test_tolerance_plot.png') pyplot.figure() calibration_bin_plot( pred,
def predict(self, X): X = copy.deepcopy(X) X_ = np.append(X, np.ones((X.shape[0], 1)), axis=1) H = expit(np.dot(X_, self.W)) output = np.dot(H, self.B) return output.argmax(axis=1)
def predict_proba(self, X): decision = self.decision_function(X) decision_2d = np.c_[-decision, decision] return expit(decision_2d)
def normalize_X(self,X): return expit(X)
def to_X_physique(self,X): """Maps mathematical X valued to physical ones""" return self.variables_lims[:,0] + self.variables_range * expit(X)
def distance(W, vocab, w1, w2): w1_id = vocab[w1][0] w2_id = vocab[w2][0] return expit(-np.dot(W[w1_id], W[w2_id]))
def np_sigmoid(gamma): return expit(gamma)
def __get_error(self, X, y, W, B): H = expit(np.dot(X, W)) y_ = np.dot(H, B).argmax(axis=1) error = 1.001 - accuracy_score(y, y_) return error