def test_levenberg_marquardt(self):
dataset = datasets.load_diabetes()
data, target = dataset.data, dataset.target
data_scaler = preprocessing.MinMaxScaler()
target_scaler = preprocessing.MinMaxScaler()
x_train, x_test, y_train, y_test = train_test_split(
data_scaler.fit_transform(data),
target_scaler.fit_transform(target),
train_size=0.85)
# Network
lmnet = algorithms.LevenbergMarquardt(connection=[
layers.SigmoidLayer(10),
layers.SigmoidLayer(40),
layers.OutputLayer(1),
],
mu_increase_factor=2,
mu=0.1,
show_epoch=10,
use_bias=False)
lmnet.train(x_train, y_train, epochs=100)
y_predict = lmnet.predict(x_test)
error = rmsle(target_scaler.inverse_transform(y_test),
target_scaler.inverse_transform(y_predict).round())
error
self.assertAlmostEqual(0.4372, error, places=4)
def test_pipeline(self):
dataset = datasets.load_diabetes()
target_scaler = preprocessing.MinMaxScaler()
x_train, x_test, y_train, y_test = train_test_split(
dataset.data,
target_scaler.fit_transform(dataset.target),
train_size=0.85)
network = algorithms.Backpropagation(
connection=[
layers.SigmoidLayer(10),
layers.SigmoidLayer(40),
layers.OutputLayer(1),
],
use_bias=True,
show_epoch=100,
verbose=False,
)
pipeline = Pipeline([
('min_max_scaler', preprocessing.MinMaxScaler()),
('backpropagation', network),
])
pipeline.fit(x_train, y_train, backpropagation__epochs=1000)
y_predict = pipeline.predict(x_test)
error = rmsle(target_scaler.inverse_transform(y_test),
target_scaler.inverse_transform(y_predict).round())
self.assertAlmostEqual(0.4481, error, places=4)
def test_hessian_diagonal(self):
dataset = datasets.load_diabetes()
data, target = dataset.data, dataset.target
input_scaler = preprocessing.StandardScaler()
target_scaler = preprocessing.StandardScaler()
x_train, x_test, y_train, y_test = cross_validation.train_test_split(
input_scaler.fit_transform(data),
target_scaler.fit_transform(target.reshape(-1, 1)),
train_size=0.8)
nw = algorithms.HessianDiagonal(connection=[
layers.SigmoidLayer(10),
layers.SigmoidLayer(20),
layers.OutputLayer(1)
],
step=1.5,
shuffle_data=False,
verbose=False,
min_eigenvalue=1e-10)
nw.train(x_train, y_train, epochs=10)
y_predict = nw.predict(x_test)
error = rmsle(target_scaler.inverse_transform(y_test),
target_scaler.inverse_transform(y_predict).round())
self.assertAlmostEqual(0.5032, error, places=4)
def test_pandas_for_bp(self):
dataset = datasets.load_diabetes()
input_scaler = preprocessing.MinMaxScaler()
target_scaler = preprocessing.MinMaxScaler()
n_features = dataset.data.shape[1]
input_columns = ['column_' + str(i) for i in range(n_features)]
pandas_data = pd.DataFrame(dataset.data, columns=input_columns)
pandas_data['target'] = target_scaler.fit_transform(dataset.target)
pandas_data[input_columns] = input_scaler.fit_transform(
pandas_data[input_columns])
x_train, x_test, y_train, y_test = train_test_split(
pandas_data[input_columns], pandas_data['target'], train_size=0.85)
bpnet = algorithms.Backpropagation(connection=[
layers.SigmoidLayer(10),
layers.SigmoidLayer(40),
layers.OutputLayer(1),
],
use_bias=True,
show_epoch=100)
bpnet.train(x_train, y_train, epochs=1000)
y_predict = bpnet.predict(x_test)
error = rmsle(target_scaler.inverse_transform(y_test),
target_scaler.inverse_transform(y_predict).round())
self.assertAlmostEqual(0.4477, error, places=4)
def test_train_different_inputs(self):
self.assertInvalidVectorTrain(
algorithms.Instar(layers.StepLayer(4) > layers.OutputLayer(1),
n_unconditioned=1,
step=1,
verbose=False),
np.array([[0, 1, -1, -1]]),
row1d=True,
)
def test_predict_different_inputs(self):
for bp_algorithm_class in self.bp_algorithms:
network = bp_algorithm_class(
layers.LinearLayer(2) > layers.OutputLayer(1),
verbose=False,
use_bias=False)
self.assertInvalidVectorPred(network,
np.array([0, 0]),
0,
row1d=True)
def test_mixture_of_experts(self):
dataset = datasets.load_diabetes()
data, target = dataset.data, dataset.target
insize, outsize = data.shape[1], 1
input_scaler = preprocessing.MinMaxScaler((-1, 1))
output_scaler = preprocessing.MinMaxScaler()
x_train, x_test, y_train, y_test = cross_validation.train_test_split(
input_scaler.fit_transform(data),
output_scaler.fit_transform(target.reshape(-1, 1)),
train_size=0.8)
n_epochs = 300
scaled_y_test = output_scaler.inverse_transform(y_test).reshape(
(y_test.size, 1))
# -------------- Train single Backpropagation -------------- #
bpnet = algorithms.Backpropagation((insize, 20, outsize),
step=0.1,
verbose=False)
bpnet.train(x_train, y_train, epochs=n_epochs)
network_output = bpnet.predict(x_test)
network_error = rmsle(output_scaler.inverse_transform(network_output),
scaled_y_test)
# -------------- Train ensemlbe -------------- #
moe = ensemble.MixtureOfExperts(
networks=[
algorithms.Backpropagation((insize, 20, outsize),
step=0.1,
verbose=False),
algorithms.Backpropagation((insize, 20, outsize),
step=0.1,
verbose=False),
],
gating_network=algorithms.Backpropagation(
layers.SoftmaxLayer(insize) > layers.OutputLayer(2),
step=0.1,
verbose=False))
moe.train(x_train, y_train, epochs=n_epochs)
ensemble_output = moe.predict(x_test)
ensemlbe_error = rmsle(
output_scaler.inverse_transform(ensemble_output), scaled_y_test)
self.assertGreater(network_error, ensemlbe_error)
def test_quasi_newton_bfgs(self):
x_train, x_test, y_train, y_test = self.data
qnnet = algorithms.QuasiNewton(
connection=[
layers.SigmoidLayer(10, init_method='ortho'),
layers.SigmoidLayer(20, init_method='ortho'),
layers.OutputLayer(1)
],
step=0.1,
shuffle_data=True,
show_epoch=20,
verbose=False,
update_function='bfgs',
h0_scale=5,
gradient_tol=1e-5,
)
qnnet.train(x_train, y_train, x_test, y_test, epochs=10)
result = qnnet.predict(x_test).round()
roc_curve_score = metrics.roc_auc_score(result, y_test)
self.assertAlmostEqual(0.92, roc_curve_score, places=2)
def test_linear_search(self):
methods = [
('golden', 0.20976),
('brent', 0.21190),
]
for method_name, valid_error in methods:
np.random.seed(self.random_seed)
dataset = datasets.load_boston()
data, target = dataset.data, dataset.target
data_scaler = preprocessing.MinMaxScaler()
target_scaler = preprocessing.MinMaxScaler()
x_train, x_test, y_train, y_test = train_test_split(
data_scaler.fit_transform(data),
target_scaler.fit_transform(target.reshape(-1, 1)),
train_size=0.85
)
cgnet = algorithms.ConjugateGradient(
connection=[
layers.SigmoidLayer(13),
layers.SigmoidLayer(50),
layers.OutputLayer(1),
],
search_method=method_name,
show_epoch=25,
optimizations=[algorithms.LinearSearch],
)
cgnet.train(x_train, y_train, epochs=100)
y_predict = cgnet.predict(x_test).round(1)
error = rmsle(target_scaler.inverse_transform(y_test),
target_scaler.inverse_transform(y_predict))
self.assertAlmostEqual(valid_error, error, places=5)
data_scaler = preprocessing.MinMaxScaler()
target_scaler = preprocessing.MinMaxScaler()
data = data_scaler.fit_transform(data)
target = target_scaler.fit_transform(target)
x_train, x_test, y_train, y_test = train_test_split(data,
target,
train_size=0.85)
cgnet = algorithms.ConjugateGradient(
connection=[
layers.SigmoidLayer(13),
layers.SigmoidLayer(50),
layers.OutputLayer(1),
],
search_method='golden',
show_epoch=25,
verbose=True,
optimizations=[algorithms.LinearSearch],
)
cgnet.train(x_train, y_train, x_test, y_test, epochs=100)
cgnet.plot_errors()
y_predict = cgnet.predict(x_test).round(1)
error = rmsle(target_scaler.inverse_transform(y_test),
target_scaler.inverse_transform(y_predict))
print("RMSLE = {}".format(error))
def setUp(self):
super(HebbRuleTestCase, self).setUp()
kwargs = {'weight': np.array([[3, 0, 0, 0]]).T}
self.conn = layers.StepLayer(4, **kwargs) > layers.OutputLayer(1)
def get_connection():
""" Generate new connections every time when we call it """
input_layer = layers.SigmoidLayer(2, weight=default_weight.copy())
output_layer = layers.OutputLayer(1)
return input_layer > output_layer
def test_handle_errors(self):
networks = [
algorithms.Backpropagation((1, 20, 1), step=0.2),
algorithms.Backpropagation((1, 20, 1), step=0.2),
]
with self.assertRaises(ValueError):
# Ivalid network (not Backpropagation)
ensemble.MixtureOfExperts(
networks=networks + [algorithms.GRNN()],
gating_network=algorithms.Backpropagation(
layers.SigmoidLayer(1) > layers.OutputLayer(3), ))
with self.assertRaises(ValueError):
# Ivalid number of outputs in third network
ensemble.MixtureOfExperts(
networks=networks +
[algorithms.Backpropagation((1, 20, 2), step=0.2)],
gating_network=algorithms.Backpropagation(
layers.SigmoidLayer(1) > layers.OutputLayer(3), ))
with self.assertRaises(ValueError):
# Ivalid gating network output layer size
ensemble.MixtureOfExperts(
networks=networks,
gating_network=algorithms.Backpropagation(
layers.SoftmaxLayer(1) > layers.OutputLayer(1), ))
with self.assertRaises(ValueError):
# Ivalid gating network input layer
ensemble.MixtureOfExperts(
networks=networks,
gating_network=algorithms.Backpropagation(
layers.SigmoidLayer(1) > layers.OutputLayer(2), ))
with self.assertRaises(ValueError):
# Ivalid gating network output layer
ensemble.MixtureOfExperts(
networks=networks,
gating_network=algorithms.Backpropagation(
layers.SoftmaxLayer(1) > layers.RoundOutputLayer(2)))
with self.assertRaises(ValueError):
# Ivalid network error function
ensemble.MixtureOfExperts(
networks=networks + [
algorithms.Backpropagation(
(1, 20, 1), step=0.2, error=rmsle)
],
gating_network=algorithms.Backpropagation(
layers.SigmoidLayer(1) > layers.OutputLayer(3), ))
with self.assertRaises(ValueError):
moe = ensemble.MixtureOfExperts(
# Ivalid gating error function
networks=networks,
gating_network=algorithms.Backpropagation(
layers.SoftmaxLayer(1) > layers.OutputLayer(2),
error=rmsle))
moe = ensemble.MixtureOfExperts(
# Ivalid gating network output layer
networks=networks,
gating_network=algorithms.Backpropagation(
layers.SoftmaxLayer(1) > layers.OutputLayer(2)))
with self.assertRaises(ValueError):
# Wrong number of train input features
moe.train(np.array([[1, 2]]), np.array([[0]]))
with self.assertRaises(ValueError):
# Wrong number of train output features
moe.train(np.array([[1]]), np.array([[0, 0]]))
