def obj_func(self, X, W, n_splits=3):
"""
error
:param X:
:param W:
:param n_splits:
:return:
"""
# ## TODO: train learner and calculate crossover Root Mean Square Error
# skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=233)
kf = KFold(n_splits=n_splits)
error = 0
kl = 0.
desired_degree = np.array([
self.sparse_degree,
] * self.n_hidden)
for train_index, test_index in kf.split(self.X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = self.T[train_index], self.T[test_index]
H = expit(np.dot(X_train, W))
B = np.dot(linalg.pinv(H), y_train)
H_ = expit(np.dot(X_test, W))
output = np.dot(H_, B)
error += np.sqrt(mean_squared_error(y_test, output))
kl_split = 0.
real_degree = H.mean(axis=0) # n_hidden dim
for p1, p2 in zip(real_degree, desired_degree):
entr = entropy([p1, 1 - p1], [p2, 1 - p2])
kl_split += entr
kl += kl_split
return kl / n_splits, error / n_splits
def obj_func_MSE(self, X, W, n_splits=3):
"""
error
:param X:
:param W:
:param n_splits:
:return:
"""
# ## TODO: train learner and calculate error of cross-over validation
skf = StratifiedKFold(n_splits=n_splits,
shuffle=True,
random_state=233)
error = 0
kl = 0.
desired_degree = np.array([
self.sparse_degree,
] * self.n_hidden)
for train_index, test_index in skf.split(self.X,
self.T.argmax(axis=1)):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = self.T[train_index], self.T[test_index]
H = expit(np.dot(X_train, W))
B = np.dot(linalg.pinv(H), y_train)
H_ = expit(np.dot(X_test, W))
output = np.dot(H_, B)
# error += (1 - accuracy_score(y_test.argmax(axis=1), output.argmax(axis=1)))
error += mean_squared_error(y_test, output)
kl_split = 0.
real_degree = H.mean(axis=0) # n_hidden dim
for p1, p2 in zip(real_degree, desired_degree):
entr = entropy([p1, 1 - p1], [p2, 1 - p2])
kl_split += entr
kl += kl_split
return kl / n_splits, error / n_splits
def obj_func_v2(self, X, W, n_splits=3):
"""
error
:param X:
:param W:
:param n_splits:
:return:
"""
# ## TODO: train learner and calculate crossover Root Mean Square Error
# skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=233)
kf = KFold(n_splits=n_splits)
error = 0
kl = 0.
desired_degree = np.array([
self.sparse_degree,
] * self.n_hidden)
for train_index, test_index in kf.split(self.X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = self.T[train_index], self.T[test_index]
H = expit(np.dot(X_train, W))
B = np.dot(linalg.pinv(H), y_train)
H_ = expit(np.dot(X_test, W))
output = np.dot(H_, B)
error += np.sqrt(mean_squared_error(y_test, output))
S = np.linalg.norm(
W, ord=1
) #np.linalg.norm(np.dot(X_test[:, :-1], B.transpose()), ord=2)/np.linalg.norm(np.dot(X_test[:, :-1], B.transpose()), ord=1)
kl += S
return kl / n_splits, error / n_splits
def predict(self, X, W=None):
X_ = np.append(X, np.ones((X.shape[0], 1)), axis=1)
if W is None:
H = expit(np.dot(X_, self.best_W))
else:
H = expit(np.dot(X_, W))
return H
def _predict_proba_lr(self, X):
prob = np.ravel(self.decision_function(X))
expit(prob, out=prob)
if prob.ndim == 1:
return np.vstack([1 - prob, prob]).T
else:
# OvR normalization, like LibLinear's predict_probability
prob /= prob.sum(axis=1).reshape((prob.shape[0], -1))
return prob
def __fitness_func(self, vars):
# cost function or optimized function
skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=233)
error = 0
X_ = np.append(self.X, np.ones((self.X.shape[0], 1)), axis=1)
W = np.asarray(vars).reshape((self.X.shape[1] + 1, self.n_hidden))
for train_index, test_index in skf.split(X_, self.y):
X_train, X_test = X_[train_index], X_[test_index]
y_train, y_test = self.y_bin[train_index], self.y_bin[test_index]
H = expit(np.dot(X_train, W))
B = np.dot(linalg.pinv(H), y_train)
H_ = expit(np.dot(X_test, W))
output = np.dot(H_, B)
error += (
1. -
accuracy_score(y_test.argmax(axis=1), output.argmax(axis=1)))
# error += mean_squared_error(y_test, output)
return error / 3.
def predict(self, X):
XX = copy.deepcopy(X)
H = None
for i in range(len(self.W)):
X_ = np.append(XX, np.ones((XX.shape[0], 1)), axis=1)
H = expit(np.dot(X_, self.W[i]))
XX = copy.deepcopy(H)
output = np.dot(H, self.B)
return output.argmax(axis=1)
def best_predict(self, X):
index = self.get_best_W_index()
W = self.W[index]
B = self.B[index]
X = copy.deepcopy(X)
X_test = np.append(X, np.ones((X.shape[0], 1)), axis=1)
H = expit(np.dot(X_test, W))
output = np.dot(H, B)
return output.argmax(axis=1)
def binary_transition_smooth(x: Union[float, np.ndarray], xthr: float, S: float = 5.0) -> Union[float, np.ndarray]:
"""
Makes a nice transition from 1 to 0 as x increases (sort of inverse sigmoid)
:param x: value to map (or array)
:param xthr: threshold value at which output is 0.5
:param S: Shape factor
:return: mapped value (or array)
"""
return 1 - expit(x / xthr * S - S)
def streaming_lr(t, x_t, y_t, betas, alpha, mu, invscaling=False):
if betas is None:
if len(x_t.shape) > 0:
betas = np.zeros(x_t.shape[0])
else:
betas = np.array(0.)
p = expit(betas.dot(x_t))
alpha = alpha / pow(t, invscaling) if invscaling else alpha
betas *= (1 - 2. * alpha * mu)
betas += alpha * (y_t - p) * x_t
return betas, alpha
def stick_breaking(Psi):
"""
Calculates the stickbreaking construction of the gaussian variables Psi
Parameters
----------
Psi: [M x K-1] Gaussian Variables used for the stick breaking
Returns
-------
Pi: [M x K] Probability matrix using the logistic function and stick breaking
"""
Pi = np.zeros((np.shape(Psi)[0], np.shape(Psi)[1] + 1))
Pi[:, 0] = expit(Psi[:, 0])
Pi[:,
1:-1] = expit(Psi[:, 1:]) * np.cumprod(1 - expit(Psi[:, 0:-1]), axis=1)
Pi[:, -1] = 1 - np.sum(Pi[:, 0:-1], axis=1)
# Check for numerical instability
if np.any(Pi[:, -1] < 0):
Pi[Pi[:, -1] < 0, -1] = 0 # Set last weight to 0
Pi /= np.sum(Pi, axis=1)[:, None] # Normalize last weight to 0
return Pi
def obj_func(self, X, W, n_splits=3):
"""
error
:param X:
:param W:
:param n_splits:
:return:
"""
# ## TODO: train learner and calculate crossover Root Mean Square Error
kf = KFold(n_splits=n_splits)
error = 0.
kl = 0.
for train_index, test_index in kf.split(self.X):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = self.T[train_index], self.T[test_index]
H = expit(np.dot(X_train, W))
B = np.dot(linalg.pinv(H), y_train)
H_ = expit(np.dot(X_test, W))
output = np.dot(H_, B)
error += np.sqrt(mean_squared_error(y_test, output))
kl += np.linalg.norm(H, ord=1)
return kl/n_splits, error/n_splits
def lr_predict(x, weights, i, learnrate, regulizer, default_value=None):
"""
Predict value using logistic regression
:param t:
:param x:
:param param:
:return:
"""
if default_value is None:
default_value = lambda: None
value = weights.get("intercept", default_value())
for key in x:
value += weights.get(key, default_value()) * x[key]
value *= (1. - 2. * learnrate * regulizer) ** i
return expit(value)
def __get_info(self, X_test, W, B, y_test=None):
H = expit(np.dot(X_test, W))
y_ = np.dot(H, B).argmax(axis=1)
# # calculate accuracy
acc = None
if y_test is not None:
acc = accuracy_score(y_test, y_)
# # calculate KL divergence
kl = 0.
desired_degree = np.array([self.sparse_degree, ] * self.n_hidden)
real_degree = H.mean(axis=0) # n_hidden dim
avg_activation = real_degree.mean()
for p1, p2 in zip(real_degree, desired_degree):
entr = entropy([p1, 1 - p1], [p2, 1 - p2])
kl += entr
return y_, acc, kl, avg_activation
def fit(self, X, y):
X, y, = copy.deepcopy(X), copy.deepcopy(y)
self.y = y
y_bin = y
self.X, self.y_bin = X, y
# # start evolving in MOEA
num_variables = (self.X.shape[1] + 1) * self.n_hidden
algorithm = NSGAII(Objectives(num_variables,
2,
self.X,
y_bin,
self.n_hidden,
sparse_degree=self.sparse_degree),
population_size=self.n_pop)
# MOEAD(Objectives(num_variables, 2, self.X, y_bin, self.n_hidden, sparse_degree=self.sparse_degree),
# population_size=self.n_pop, neighborhood_size=int(self.n_pop/10)) # delta=0.5, eta=0.8
algorithm.run(self.max_iter)
self.evo_result = algorithm.result
print('total solution:', algorithm.result.__len__())
nondom_result = nondominated(algorithm.result)
print('nondominated solution:', nondom_result.__len__())
self.nondom_solution = nondom_result
self.W = []
self.B = []
for i in range(nondom_result.__len__()):
s = nondom_result[i]
W = np.asarray(s.variables).reshape(self.X.shape[1] + 1,
self.n_hidden)
X_ = np.append(self.X, np.ones((self.X.shape[0], 1)), axis=1)
H = expit(np.dot(X_, W))
B = np.dot(linalg.pinv(H), y_bin)
self.W.append(W)
self.B.append(B)
real_degree = H.mean(axis=0) # n_hidden dim
avg_activation = real_degree.mean()
print('NO.', i, ' obj:', s.objectives, 'AVG activation:',
avg_activation)
self.W = np.asarray(self.W)
self.B = np.asarray(self.B)
# # best W/B
best_index = self.get_best_index()
self.best_W = self.W[best_index]
self.best_B = self.B[best_index]
return self
def fit(self, X, y):
X, y, = copy.deepcopy(X), copy.deepcopy(y)
self.y = y
y_bin = self.one2array(y, np.unique(y).shape[0])
self.classes_ = np.arange(y_bin.shape[1])
self.n_classes_ = self.classes_.__len__()
self.X, self.y_bin = X, y_bin
# # start evolving in MOEA
num_variables = (self.X.shape[1] + 1) * self.n_hidden
algorithm = MOEAD(Objectives(num_variables,
2,
self.X,
y_bin,
self.n_hidden,
sparse_degree=self.sparse_degree),
population_size=self.n_pop,
neighborhood_size=5)
algorithm.run(self.max_iter)
self.evo_result = algorithm.result
print('total solution:', algorithm.result.__len__())
result = nondominated(algorithm.result)
print('nondominated solution:', result.__len__())
self.solution = result
self.W = []
self.B = []
self.voting_weight = []
for s in result:
W = np.asarray(s.variables).reshape(self.X.shape[1] + 1,
self.n_hidden)
X_ = np.append(self.X, np.ones((self.X.shape[0], 1)), axis=1)
H = expit(np.dot(X_, W))
B = np.dot(linalg.pinv(H), y_bin)
voting_w_ = 1. / (s.objectives[0] + 10e-5) * self.mu + 1. / (
s.objectives[1] + 10e-5) * (1 - self.mu)
self.voting_weight.append(voting_w_)
self.W.append(W)
self.B.append(B)
self.voting_weight = np.asarray(self.voting_weight)
self.W = np.asarray(self.W)
self.B = np.asarray(self.B)
return self
def fit(self, X, y):
X, y, = copy.deepcopy(X), copy.deepcopy(y)
self.y = y
y_bin = self.one2array(y, np.unique(y).shape[0])
self.classes_ = np.arange(y_bin.shape[1])
self.n_classes_ = self.classes_.__len__()
self.X, self.y_bin = X, y_bin
bounds = np.array([
[-1., 1.],
] * (self.X.shape[1] + 1) * self.n_hidden)
de = DiffEvolOptimizer(self.__fitness_func,
bounds,
self.n_pop,
F=0.7,
C=0.5)
solution, fitness = de.optimize(ngen=self.max_iter)
self.W = np.asarray(solution).reshape(
(self.X.shape[1] + 1, self.n_hidden))
X_ = np.append(self.X, np.ones((self.X.shape[0], 1)), axis=1)
H = expit(np.dot(X_, self.W))
self.B = np.dot(linalg.pinv(H), y_bin)
return self
def predict(self, X):
X_ = np.append(X, np.ones((X.shape[0], 1)), axis=1)
output = expit(np.dot(X_, self.orth_W))
return output
def test_zero(self):
self.assertAlmostEqual(expit(0.), 0.5, places=5)
def test_minus_inf(self):
self.assertAlmostEqual(expit(-100), 0., places=5)
from sklearntools.validation import plot_tolerance, plot_roc,\
calibration_bin_plot, plot_roc_auc_for_bins
if __name__ == '__main__':
from sklearntools.validation import plot_curve_auc, roc_curve
from sklearn.linear_model.logistic import LogisticRegression
from scipy.special._ufuncs import expit
import numpy as np
from matplotlib import pyplot
np.random.seed(1)
m = 1000
n = 5
X = np.random.normal(size=(m, n))
beta = np.random.normal(size=n)
y = np.random.binomial(n=1, p=expit(np.dot(X, beta)))
model = LogisticRegression().fit(X, y)
pred = model.predict_proba(X)[:, 1]
pyplot.figure()
plot_roc(y, pred, name='test_model')
pyplot.savefig('test_roc_plot.png')
pyplot.figure()
plot_tolerance(y, pred, name='test_model', normalize=True)
pyplot.savefig('test_tolerance_plot.png')
pyplot.figure()
calibration_bin_plot(
pred,
def predict(self, X):
X = copy.deepcopy(X)
X_ = np.append(X, np.ones((X.shape[0], 1)), axis=1)
H = expit(np.dot(X_, self.W))
output = np.dot(H, self.B)
return output.argmax(axis=1)
def predict_proba(self, X):
decision = self.decision_function(X)
decision_2d = np.c_[-decision, decision]
return expit(decision_2d)
def normalize_X(self,X):
return expit(X)
def to_X_physique(self,X):
"""Maps mathematical X valued to physical ones"""
return self.variables_lims[:,0] + self.variables_range * expit(X)
def distance(W, vocab, w1, w2):
w1_id = vocab[w1][0]
w2_id = vocab[w2][0]
return expit(-np.dot(W[w1_id], W[w2_id]))
def np_sigmoid(gamma):
return expit(gamma)
def __get_error(self, X, y, W, B):
H = expit(np.dot(X, W))
y_ = np.dot(H, B).argmax(axis=1)
error = 1.001 - accuracy_score(y, y_)
return error