Ejemplo n.º 1
0
 def obj_func(self, X, W, n_splits=3):
     """
     error
     :param X:
     :param W:
     :param n_splits:
     :return:
     """
     # ## TODO: train learner and calculate crossover Root Mean Square Error
     # skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=233)
     kf = KFold(n_splits=n_splits)
     error = 0
     kl = 0.
     desired_degree = np.array([
         self.sparse_degree,
     ] * self.n_hidden)
     for train_index, test_index in kf.split(self.X):
         X_train, X_test = X[train_index], X[test_index]
         y_train, y_test = self.T[train_index], self.T[test_index]
         H = expit(np.dot(X_train, W))
         B = np.dot(linalg.pinv(H), y_train)
         H_ = expit(np.dot(X_test, W))
         output = np.dot(H_, B)
         error += np.sqrt(mean_squared_error(y_test, output))
         kl_split = 0.
         real_degree = H.mean(axis=0)  # n_hidden dim
         for p1, p2 in zip(real_degree, desired_degree):
             entr = entropy([p1, 1 - p1], [p2, 1 - p2])
             kl_split += entr
         kl += kl_split
     return kl / n_splits, error / n_splits
Ejemplo n.º 2
0
 def obj_func_MSE(self, X, W, n_splits=3):
     """
     error
     :param X:
     :param W:
     :param n_splits:
     :return:
     """
     # ## TODO: train learner and calculate error of cross-over validation
     skf = StratifiedKFold(n_splits=n_splits,
                           shuffle=True,
                           random_state=233)
     error = 0
     kl = 0.
     desired_degree = np.array([
         self.sparse_degree,
     ] * self.n_hidden)
     for train_index, test_index in skf.split(self.X,
                                              self.T.argmax(axis=1)):
         X_train, X_test = X[train_index], X[test_index]
         y_train, y_test = self.T[train_index], self.T[test_index]
         H = expit(np.dot(X_train, W))
         B = np.dot(linalg.pinv(H), y_train)
         H_ = expit(np.dot(X_test, W))
         output = np.dot(H_, B)
         # error += (1 - accuracy_score(y_test.argmax(axis=1), output.argmax(axis=1)))
         error += mean_squared_error(y_test, output)
         kl_split = 0.
         real_degree = H.mean(axis=0)  # n_hidden dim
         for p1, p2 in zip(real_degree, desired_degree):
             entr = entropy([p1, 1 - p1], [p2, 1 - p2])
             kl_split += entr
         kl += kl_split
     return kl / n_splits, error / n_splits
Ejemplo n.º 3
0
 def obj_func_v2(self, X, W, n_splits=3):
     """
     error
     :param X:
     :param W:
     :param n_splits:
     :return:
     """
     # ## TODO: train learner and calculate crossover Root Mean Square Error
     # skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=233)
     kf = KFold(n_splits=n_splits)
     error = 0
     kl = 0.
     desired_degree = np.array([
         self.sparse_degree,
     ] * self.n_hidden)
     for train_index, test_index in kf.split(self.X):
         X_train, X_test = X[train_index], X[test_index]
         y_train, y_test = self.T[train_index], self.T[test_index]
         H = expit(np.dot(X_train, W))
         B = np.dot(linalg.pinv(H), y_train)
         H_ = expit(np.dot(X_test, W))
         output = np.dot(H_, B)
         error += np.sqrt(mean_squared_error(y_test, output))
         S = np.linalg.norm(
             W, ord=1
         )  #np.linalg.norm(np.dot(X_test[:, :-1], B.transpose()), ord=2)/np.linalg.norm(np.dot(X_test[:, :-1], B.transpose()), ord=1)
         kl += S
     return kl / n_splits, error / n_splits
Ejemplo n.º 4
0
 def predict(self, X, W=None):
     X_ = np.append(X, np.ones((X.shape[0], 1)), axis=1)
     if W is None:
         H = expit(np.dot(X_, self.best_W))
     else:
         H = expit(np.dot(X_, W))
     return H
Ejemplo n.º 5
0
 def _predict_proba_lr(self, X):
     prob = np.ravel(self.decision_function(X))
     expit(prob, out=prob)
     if prob.ndim == 1:
         return np.vstack([1 - prob, prob]).T
     else:
         # OvR normalization, like LibLinear's predict_probability
         prob /= prob.sum(axis=1).reshape((prob.shape[0], -1))
         return prob
Ejemplo n.º 6
0
 def __fitness_func(self, vars):
     # cost function or optimized function
     skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=233)
     error = 0
     X_ = np.append(self.X, np.ones((self.X.shape[0], 1)), axis=1)
     W = np.asarray(vars).reshape((self.X.shape[1] + 1, self.n_hidden))
     for train_index, test_index in skf.split(X_, self.y):
         X_train, X_test = X_[train_index], X_[test_index]
         y_train, y_test = self.y_bin[train_index], self.y_bin[test_index]
         H = expit(np.dot(X_train, W))
         B = np.dot(linalg.pinv(H), y_train)
         H_ = expit(np.dot(X_test, W))
         output = np.dot(H_, B)
         error += (
             1. -
             accuracy_score(y_test.argmax(axis=1), output.argmax(axis=1)))
         # error += mean_squared_error(y_test, output)
     return error / 3.
Ejemplo n.º 7
0
 def predict(self, X):
     XX = copy.deepcopy(X)
     H = None
     for i in range(len(self.W)):
         X_ = np.append(XX, np.ones((XX.shape[0], 1)), axis=1)
         H = expit(np.dot(X_, self.W[i]))
         XX = copy.deepcopy(H)
     output = np.dot(H, self.B)
     return output.argmax(axis=1)
Ejemplo n.º 8
0
 def best_predict(self, X):
     index = self.get_best_W_index()
     W = self.W[index]
     B = self.B[index]
     X = copy.deepcopy(X)
     X_test = np.append(X, np.ones((X.shape[0], 1)), axis=1)
     H = expit(np.dot(X_test, W))
     output = np.dot(H, B)
     return output.argmax(axis=1)
Ejemplo n.º 9
0
def binary_transition_smooth(x: Union[float, np.ndarray], xthr: float, S: float = 5.0) -> Union[float, np.ndarray]:
    """
    Makes a nice transition from 1 to 0 as x increases (sort of inverse sigmoid)
    :param x: value to map (or array)
    :param xthr: threshold value at which output is 0.5
    :param S: Shape factor
    :return: mapped value (or array)

    """
    return 1 - expit(x / xthr * S - S)
def streaming_lr(t, x_t, y_t, betas, alpha, mu, invscaling=False):
    if betas is None:
        if len(x_t.shape) > 0:
            betas = np.zeros(x_t.shape[0])
        else:
            betas = np.array(0.)
    p = expit(betas.dot(x_t))
    alpha = alpha / pow(t, invscaling) if invscaling else alpha
    betas *= (1 - 2. * alpha * mu)
    betas += alpha * (y_t - p) * x_t
    return betas, alpha
Ejemplo n.º 11
0
def stick_breaking(Psi):
    """
    Calculates the stickbreaking construction of the gaussian variables Psi
    Parameters
    ----------
    Psi: [M x K-1] Gaussian Variables used for the stick breaking

    Returns
    -------
    Pi: [M x K] Probability matrix using the logistic function and stick breaking
    """
    Pi = np.zeros((np.shape(Psi)[0], np.shape(Psi)[1] + 1))
    Pi[:, 0] = expit(Psi[:, 0])
    Pi[:,
       1:-1] = expit(Psi[:, 1:]) * np.cumprod(1 - expit(Psi[:, 0:-1]), axis=1)
    Pi[:, -1] = 1 - np.sum(Pi[:, 0:-1], axis=1)
    # Check for numerical instability
    if np.any(Pi[:, -1] < 0):
        Pi[Pi[:, -1] < 0, -1] = 0  # Set last weight to 0
        Pi /= np.sum(Pi, axis=1)[:, None]  # Normalize last weight to 0

    return Pi
Ejemplo n.º 12
0
 def obj_func(self, X, W, n_splits=3):
     """
     error
     :param X:
     :param W:
     :param n_splits:
     :return:
     """
     # ## TODO: train learner and calculate crossover Root Mean Square Error
     kf = KFold(n_splits=n_splits)
     error = 0.
     kl = 0.
     for train_index, test_index in kf.split(self.X):
         X_train, X_test = X[train_index], X[test_index]
         y_train, y_test = self.T[train_index], self.T[test_index]
         H = expit(np.dot(X_train, W))
         B = np.dot(linalg.pinv(H), y_train)
         H_ = expit(np.dot(X_test, W))
         output = np.dot(H_, B)
         error += np.sqrt(mean_squared_error(y_test, output))
         kl += np.linalg.norm(H, ord=1)
     return kl/n_splits, error/n_splits
def lr_predict(x, weights, i, learnrate, regulizer, default_value=None):
    """
    Predict value using logistic regression
    :param t:
    :param x:
    :param param:
    :return:
    """
    if default_value is None:
        default_value = lambda: None
    value = weights.get("intercept", default_value())
    for key in x:
        value += weights.get(key, default_value()) * x[key]
    value *= (1. - 2. * learnrate * regulizer) ** i
    return expit(value)
Ejemplo n.º 14
0
 def __get_info(self, X_test, W, B, y_test=None):
     H = expit(np.dot(X_test, W))
     y_ = np.dot(H, B).argmax(axis=1)
     # # calculate accuracy
     acc = None
     if y_test is not None:
         acc = accuracy_score(y_test, y_)
     # # calculate KL divergence
     kl = 0.
     desired_degree = np.array([self.sparse_degree, ] * self.n_hidden)
     real_degree = H.mean(axis=0)  # n_hidden dim
     avg_activation = real_degree.mean()
     for p1, p2 in zip(real_degree, desired_degree):
         entr = entropy([p1, 1 - p1], [p2, 1 - p2])
         kl += entr
     return y_, acc, kl, avg_activation
Ejemplo n.º 15
0
    def fit(self, X, y):
        X, y, = copy.deepcopy(X), copy.deepcopy(y)
        self.y = y
        y_bin = y
        self.X, self.y_bin = X, y
        # # start evolving in MOEA
        num_variables = (self.X.shape[1] + 1) * self.n_hidden
        algorithm = NSGAII(Objectives(num_variables,
                                      2,
                                      self.X,
                                      y_bin,
                                      self.n_hidden,
                                      sparse_degree=self.sparse_degree),
                           population_size=self.n_pop)

        # MOEAD(Objectives(num_variables, 2, self.X, y_bin, self.n_hidden, sparse_degree=self.sparse_degree),
        #               population_size=self.n_pop, neighborhood_size=int(self.n_pop/10))  # delta=0.5, eta=0.8
        algorithm.run(self.max_iter)
        self.evo_result = algorithm.result
        print('total solution:', algorithm.result.__len__())
        nondom_result = nondominated(algorithm.result)
        print('nondominated solution:', nondom_result.__len__())
        self.nondom_solution = nondom_result
        self.W = []
        self.B = []
        for i in range(nondom_result.__len__()):
            s = nondom_result[i]
            W = np.asarray(s.variables).reshape(self.X.shape[1] + 1,
                                                self.n_hidden)
            X_ = np.append(self.X, np.ones((self.X.shape[0], 1)), axis=1)
            H = expit(np.dot(X_, W))
            B = np.dot(linalg.pinv(H), y_bin)
            self.W.append(W)
            self.B.append(B)
            real_degree = H.mean(axis=0)  # n_hidden dim
            avg_activation = real_degree.mean()
            print('NO.', i, '  obj:', s.objectives, 'AVG activation:',
                  avg_activation)
        self.W = np.asarray(self.W)
        self.B = np.asarray(self.B)
        # # best W/B
        best_index = self.get_best_index()
        self.best_W = self.W[best_index]
        self.best_B = self.B[best_index]
        return self
Ejemplo n.º 16
0
 def fit(self, X, y):
     X, y, = copy.deepcopy(X), copy.deepcopy(y)
     self.y = y
     y_bin = self.one2array(y, np.unique(y).shape[0])
     self.classes_ = np.arange(y_bin.shape[1])
     self.n_classes_ = self.classes_.__len__()
     self.X, self.y_bin = X, y_bin
     # # start evolving in MOEA
     num_variables = (self.X.shape[1] + 1) * self.n_hidden
     algorithm = MOEAD(Objectives(num_variables,
                                  2,
                                  self.X,
                                  y_bin,
                                  self.n_hidden,
                                  sparse_degree=self.sparse_degree),
                       population_size=self.n_pop,
                       neighborhood_size=5)
     algorithm.run(self.max_iter)
     self.evo_result = algorithm.result
     print('total solution:', algorithm.result.__len__())
     result = nondominated(algorithm.result)
     print('nondominated solution:', result.__len__())
     self.solution = result
     self.W = []
     self.B = []
     self.voting_weight = []
     for s in result:
         W = np.asarray(s.variables).reshape(self.X.shape[1] + 1,
                                             self.n_hidden)
         X_ = np.append(self.X, np.ones((self.X.shape[0], 1)), axis=1)
         H = expit(np.dot(X_, W))
         B = np.dot(linalg.pinv(H), y_bin)
         voting_w_ = 1. / (s.objectives[0] + 10e-5) * self.mu + 1. / (
             s.objectives[1] + 10e-5) * (1 - self.mu)
         self.voting_weight.append(voting_w_)
         self.W.append(W)
         self.B.append(B)
     self.voting_weight = np.asarray(self.voting_weight)
     self.W = np.asarray(self.W)
     self.B = np.asarray(self.B)
     return self
Ejemplo n.º 17
0
 def fit(self, X, y):
     X, y, = copy.deepcopy(X), copy.deepcopy(y)
     self.y = y
     y_bin = self.one2array(y, np.unique(y).shape[0])
     self.classes_ = np.arange(y_bin.shape[1])
     self.n_classes_ = self.classes_.__len__()
     self.X, self.y_bin = X, y_bin
     bounds = np.array([
         [-1., 1.],
     ] * (self.X.shape[1] + 1) * self.n_hidden)
     de = DiffEvolOptimizer(self.__fitness_func,
                            bounds,
                            self.n_pop,
                            F=0.7,
                            C=0.5)
     solution, fitness = de.optimize(ngen=self.max_iter)
     self.W = np.asarray(solution).reshape(
         (self.X.shape[1] + 1, self.n_hidden))
     X_ = np.append(self.X, np.ones((self.X.shape[0], 1)), axis=1)
     H = expit(np.dot(X_, self.W))
     self.B = np.dot(linalg.pinv(H), y_bin)
     return self
Ejemplo n.º 18
0
 def predict(self, X):
     X_ = np.append(X, np.ones((X.shape[0], 1)), axis=1)
     output = expit(np.dot(X_, self.orth_W))
     return output
Ejemplo n.º 19
0
 def test_zero(self):
     self.assertAlmostEqual(expit(0.), 0.5, places=5)
Ejemplo n.º 20
0
 def test_minus_inf(self):
     self.assertAlmostEqual(expit(-100), 0., places=5)
from sklearntools.validation import plot_tolerance, plot_roc,\
    calibration_bin_plot, plot_roc_auc_for_bins

if __name__ == '__main__':
    from sklearntools.validation import plot_curve_auc, roc_curve
    from sklearn.linear_model.logistic import LogisticRegression
    from scipy.special._ufuncs import expit
    import numpy as np
    from matplotlib import pyplot
    np.random.seed(1)
    m = 1000
    n = 5

    X = np.random.normal(size=(m, n))
    beta = np.random.normal(size=n)
    y = np.random.binomial(n=1, p=expit(np.dot(X, beta)))

    model = LogisticRegression().fit(X, y)
    pred = model.predict_proba(X)[:, 1]

    pyplot.figure()
    plot_roc(y, pred, name='test_model')
    pyplot.savefig('test_roc_plot.png')

    pyplot.figure()
    plot_tolerance(y, pred, name='test_model', normalize=True)
    pyplot.savefig('test_tolerance_plot.png')

    pyplot.figure()
    calibration_bin_plot(
        pred,
Ejemplo n.º 22
0
 def predict(self, X):
     X = copy.deepcopy(X)
     X_ = np.append(X, np.ones((X.shape[0], 1)), axis=1)
     H = expit(np.dot(X_, self.W))
     output = np.dot(H, self.B)
     return output.argmax(axis=1)
Ejemplo n.º 23
0
 def predict_proba(self, X):
     decision = self.decision_function(X)
     decision_2d = np.c_[-decision, decision]
     return expit(decision_2d)
Ejemplo n.º 24
0
 def normalize_X(self,X):
     return expit(X)
Ejemplo n.º 25
0
 def to_X_physique(self,X):
     """Maps mathematical X valued to physical ones"""
     return self.variables_lims[:,0] + self.variables_range * expit(X)
Ejemplo n.º 26
0
def distance(W, vocab, w1, w2):
    w1_id = vocab[w1][0]
    w2_id = vocab[w2][0]

    return expit(-np.dot(W[w1_id], W[w2_id]))
Ejemplo n.º 27
0
def np_sigmoid(gamma):
    return expit(gamma)
Ejemplo n.º 28
0
 def __get_error(self, X, y, W, B):
     H = expit(np.dot(X, W))
     y_ = np.dot(H, B).argmax(axis=1)
     error = 1.001 - accuracy_score(y, y_)
     return error