def test_plot_errors_no_batch(self):
x_train, x_test, y_train, y_test = simple_classification()
optimizer = algorithms.Adadelta(self.network, batch_size=None)
optimizer.train(x_train, y_train, x_test, y_test, epochs=10)
optimizer.plot_errors(show=False)
return plt.gcf()
def test_not_tarining_data_to_plot(self):
optimizer = algorithms.Adadelta(self.network)
with warnings.catch_warnings(record=True) as warns:
optimizer.plot_errors()
self.assertEqual(1, len(warns))
self.assertEqual(
str(warns[-1].message),
"There is no data to plot")
def test_simple_adadelta(self):
x_train, _, y_train, _ = simple_classification()
mnet = algorithms.Adadelta(
(10, 20, 1),
step=2.,
batch_size='full',
verbose=False,
decay=0.95,
epsilon=1e-5,
)
mnet.train(x_train, y_train, epochs=100)
self.assertAlmostEqual(0.04, mnet.errors.last(), places=2)
def test_simple_adadelta(self):
x_train, x_test, y_train, y_test = simple_classification()
mnet = algorithms.Adadelta(
(10, 20, 1),
batch_size='full',
verbose=False,
decay=0.95,
epsilon=1e-5,
)
mnet.train(x_train, y_train, x_test, y_test, epochs=100)
self.assertGreater(0.05, mnet.errors.last())
self.assertGreater(0.15, mnet.validation_errors.last())
def test_adadelta(self):
x_train, x_test, y_train, y_test = simple_classification()
optimizer = algorithms.Adadelta(
self.network,
batch_size=None,
verbose=False,
rho=0.95,
epsilon=1e-5,
)
optimizer.train(x_train, y_train, x_test, y_test, epochs=100)
self.assertGreater(0.05, optimizer.errors.train[-1])
self.assertGreater(0.15, optimizer.errors.valid[-1])
def test_mixture_of_experts_repr(self):
moe = algorithms.MixtureOfExperts(
networks=[
algorithms.Momentum((3, 2, 1)),
algorithms.GradientDescent((3, 2, 1)),
],
gating_network=algorithms.Adadelta(
layers.Input(3) > layers.Softmax(2), ))
moe_repr = str(moe)
self.assertIn('MixtureOfExperts', moe_repr)
self.assertIn('Momentum', moe_repr)
self.assertIn('GradientDescent', moe_repr)
self.assertIn('Adadelta', moe_repr)
def test_plot_errors_show_triggered_automatically(self):
x_train, x_test, y_train, y_test = simple_classification()
optimizer = algorithms.Adadelta(
self.network,
shuffle_data=True,
batch_size=10,
)
optimizer.train(x_train, y_train, epochs=100)
events = []
def mocked_show(*args, **kwargs):
events.append('show')
with mock.patch('matplotlib.pyplot.show', side_effect=mocked_show):
optimizer.plot_errors(show=True)
self.assertSequenceEqual(events, ['show'])
return plt.gcf()
def select_algorithm(self, algorithm, options=None):
try:
self.network = algorithms.LevenbergMarquardt(self.layers)
opt = options
print(opt[1])
print("Wybrano optymalizator: " + str(algorithm))
except RecursionError:
print("Problem rekursji")
return None
if algorithm == 'GradientDescent':
self.network = algorithms.GradientDescent(self.layers)
if algorithm == 'LevenbergMarquardt':
self.network = algorithms.LevenbergMarquardt(connection=self.layers, mu=opt[0], mu_update_factor=opt[1])
if algorithm == 'Adam':
self.network = algorithms.Adam(self.layers)
if algorithm == 'QuasiNewton':
self.network = algorithms.QuasiNewton(self.layers)
if algorithm == 'Quickprop':
self.network = algorithms.Quickprop(self.layers)
if algorithm == 'MinibatchGradientDescent':
self.network = algorithms.MinibatchGradientDescent(self.layers)
if algorithm == 'ConjugateGradient':
self.network = algorithms.ConjugateGradient(self.layers)
if algorithm == 'Hessian':
self.network = algorithms.Hessian(self.layers)
if algorithm == 'HessianDiagonal':
self.network = algorithms.HessianDiagonal(self.layers)
if algorithm == 'Momentum':
self.network = algorithms.Momentum(self.layers)
if algorithm == 'RPROP':
self.network = algorithms.RPROP(self.layers)
if algorithm == 'IRPROPPlus':
self.network = algorithms.IRPROPPlus(self.layers)
if algorithm == 'Adadelta':
self.network = algorithms.Adadelta(self.layers)
if algorithm == 'Adagrad':
self.network = algorithms.Adagrad(self.layers)
if algorithm == 'RMSProp':
self.network = algorithms.RMSProp(self.layers)
if algorithm == 'Adamax':
self.network = algorithms.Adamax(self.layers)
data = data.reshape((n_samples, 1, 28, 28))
x_train, x_test, y_train, y_test = cross_validation.train_test_split(
data.astype(np.float32), target.astype(np.float32), train_size=(6 / 7.))
network = algorithms.Adadelta(
[
layers.Convolution((32, 1, 3, 3)),
layers.Relu(),
layers.Convolution((48, 32, 3, 3)),
layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Dropout(0.2),
layers.Reshape(),
layers.Relu(6912),
layers.Dropout(0.3),
layers.Softmax(200),
layers.ArgmaxOutput(10),
],
error='categorical_crossentropy',
step=1.0,
verbose=True,
shuffle_data=True,
epochs_step_minimizator=8,
addons=[algorithms.SimpleStepMinimization],
)
network.architecture()
network.train(x_train, y_train, x_test, y_test, epochs=6)
y_predicted = network.predict(x_test)
y_test_labels = np.asarray(y_test.argmax(axis=1)).reshape(len(y_test))
network = algorithms.Adadelta([
layers.Convolution((64, 3, 3, 3), border_mode='full'),
layers.Relu(),
layers.Convolution((64, 64, 3, 3)),
layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Dropout(proba=0.25),
layers.Convolution((128, 64, 3, 3), border_mode='full'),
layers.Relu(),
layers.Convolution((128, 128, 3, 3)),
layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Dropout(proba=0.25),
layers.Convolution((256, 128, 3, 3), border_mode='full'),
layers.Relu(),
layers.Convolution((256, 256, 3, 3)),
layers.Relu(),
layers.Convolution((256, 256, 1, 1)),
layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Dropout(proba=0.25),
layers.Reshape(),
layers.Relu(256 * 4 * 4),
layers.Dropout(proba=0.5),
layers.Relu(1024),
layers.Dropout(proba=0.5),
layers.Softmax(1024),
layers.ArgmaxOutput(10),
],
error='categorical_crossentropy',
step=0.25,
shuffle_data=True,
verbose=True,
epochs_step_minimizator=5,
addons=[algorithms.SimpleStepMinimization])
conv_autoencoder.train(x_unlabeled_4d, x_unlabeled,
x_labeled_4d, x_labeled, epochs=10)
x_labeled_encoded = encoder.output(x_labeled_4d).eval()
x_unlabeled_encoded = encoder.output(x_unlabeled_4d).eval()
classifier_network = layers.join(
layers.PRelu(512),
layers.Dropout(0.25),
layers.Softmax(10),
)
encoder_classifier = algorithms.Adadelta(
layers.Input(encoder.output_shape) > classifier_network,
verbose=True,
step=0.05,
shuffle_data=True,
batch_size=64,
error='categorical_crossentropy',
)
encoder_classifier.architecture()
encoder_classifier.train(x_labeled_encoded, y_labeled,
x_unlabeled_encoded, y_unlabeled, epochs=100)
classifier = algorithms.MinibatchGradientDescent(
encoder > classifier_network,
verbose=True,
step=0.01,
shuffle_data=True,
batch_size=64,
error='categorical_crossentropy',
)
network = algorithms.Adadelta(
[
layers.Input((1, 28, 28)),
layers.Convolution((32, 3, 3)) > layers.BatchNorm() > layers.Relu(),
layers.Convolution((48, 3, 3)) > layers.BatchNorm() > layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Convolution((64, 3, 3)) > layers.BatchNorm() > layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Reshape(),
layers.Linear(1024) > layers.BatchNorm() > layers.Relu(),
layers.Softmax(10),
],
# Using categorical cross-entropy as a loss function
error='categorical_crossentropy',
# Min-batch size
batch_size=128,
# Learning rate. We can allow high values
# since we are using Batch Normalization
step=1.0,
# Shows information about algorithm and
# training progress in terminal
verbose=True,
# Randomly shuffles training dataset before every epoch
shuffle_data=True,
# Step decay algorithm minimizes learning step
# monotonically after each iteration.
addons=[algorithms.StepDecay],
# Parameter controls step redution frequency. The higher
# the value the slower step parameter decreases.
reduction_freq=8,
)
x_test -= mean
x_test /= std
return x_train, x_test, y_train, y_test
network = algorithms.Adadelta(
[
layers.Input((1, 28, 28)),
layers.Convolution((32, 3, 3)) > layers.BatchNorm() > layers.Relu(),
layers.Convolution((48, 3, 3)) > layers.BatchNorm() > layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Convolution((64, 3, 3)) > layers.BatchNorm() > layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Reshape(),
layers.Linear(1024) > layers.BatchNorm() > layers.Relu(),
layers.Softmax(10),
],
error='categorical_crossentropy',
step=1.0,
verbose=True,
shuffle_data=True,
reduction_freq=8,
addons=[algorithms.StepDecay],
)
network.architecture()
x_train, x_test, y_train, y_test = load_data()
network.train(x_train, y_train, x_test, y_test, epochs=2)
y_predicted = network.predict(x_test).argmax(axis=1)
x_test /= std
target_scaler = OneHotEncoder()
y_train = target_scaler.fit_transform(y_train.reshape((-1, 1))).todense()
y_test = target_scaler.transform(y_test.reshape((-1, 1))).todense()
network = algorithms.Adadelta(
[
layers.Input((3, 32, 32)),
layers.Convolution((64, 3, 3)) > layers.BatchNorm() > layers.PRelu(),
layers.Convolution((64, 3, 3)) > layers.BatchNorm() > layers.PRelu(),
layers.MaxPooling((2, 2)),
layers.Convolution((128, 3, 3)) > layers.BatchNorm() > layers.PRelu(),
layers.Convolution((128, 3, 3)) > layers.BatchNorm() > layers.PRelu(),
layers.MaxPooling((2, 2)),
layers.Reshape(),
layers.Linear(1024) > layers.BatchNorm() > layers.PRelu(),
layers.Linear(1024) > layers.BatchNorm() > layers.PRelu(),
layers.Softmax(10),
],
error='categorical_crossentropy',
step=0.25,
shuffle_data=True,
batch_size=128,
verbose=True,
)
network.architecture()
network.train(x_train, y_train, x_test, y_test, epochs=20)
y_predicted = network.predict(x_test).argmax(axis=1)
y_test_labels = np.asarray(y_test.argmax(axis=1)).reshape(len(y_test))
x_labeled_4d,
x_labeled,
epochs=10)
classifier_structure, _ = surgery.cut_along_lines(conv_autoencoder)
x_labeled_encoded = classifier_structure.output(x_labeled_4d).eval()
x_unlabeled_encoded = classifier_structure.output(x_unlabeled_4d).eval()
linear_classifier = algorithms.Adadelta(
[
layers.Input(classifier_structure.output_shape),
layers.PRelu(512),
layers.Dropout(0.25),
layers.Softmax(10),
],
verbose=True,
step=0.05,
shuffle_data=True,
batch_size=128,
error='categorical_crossentropy',
)
linear_classifier.architecture()
linear_classifier.train(x_labeled_encoded,
y_labeled,
x_unlabeled_encoded,
y_unlabeled,
epochs=100)
classification_layer = surgery.cut(linear_classifier, start=1, end=4)
classifier_structure = surgery.sew_together(
