def test_mnist_start_large(self):
data = self.mnist_data
input_layer = InputLayer(784)
hidden_1 = DuelStateReluLayer(input_layer, 200, session=self.session, inactive_nodes_to_leave=200)
output = HiddenLayer(hidden_1, self.MNIST_OUTPUT_NODES, session=self.session)
trainer = CategoricalTrainer(output, 0.1)
end_epoch = data.train.epochs_completed + 5
print(trainer.accuracy(data.test.features, data.test.labels))
while data.train.epochs_completed <= end_epoch:
train_x, train_y = data.train.next_batch(100)
trainer.train(train_x, train_y)
accuracy, cost = trainer.accuracy(data.test.features, data.test.labels)
print(accuracy, cost)
# print(output.active_nodes())
print(hidden_1.active_nodes())
self.assertGreater(accuracy, 90.)
# self.assertEqual(output.active_nodes(), self.MNIST_OUTPUT_NODES, msg='expect all output nodes to be active')
self.assertLess(hidden_1.active_nodes(), hidden_1.output_nodes, msg='expect not all hidden nodes to be active')
def test_mnist_start_large(self):
data = self.mnist_data
input_layer = InputLayer(784)
hidden_1 = DuelStateReluLayer(input_layer, 200, session=self.session, inactive_nodes_to_leave=200)
output = Layer(hidden_1, self.MNIST_OUTPUT_NODES, session=self.session)
trainer = CategoricalTrainer(output, 0.1)
end_epoch = data.train.epochs_completed + 5
print(trainer.accuracy(data.test.images, data.test.labels))
while data.train.epochs_completed <= end_epoch:
train_x, train_y = data.train.next_batch(100)
trainer.train(train_x, train_y)
accuracy, cost = trainer.accuracy(data.test.images, data.test.labels)
print(accuracy, cost)
# print(output.active_nodes())
print(hidden_1.active_nodes())
self.assertGreater(accuracy, 90.)
# self.assertEqual(output.active_nodes(), self.MNIST_OUTPUT_NODES, msg='expect all output nodes to be active')
self.assertLess(hidden_1.active_nodes(), hidden_1.output_nodes, msg='expect not all hidden nodes to be active')
def test_accuracy_bug(self):
import tensor_dynamic.data.input_data as mnist
data = mnist.read_data_sets("../data/MNIST_data", one_hot=True)
inputs = tf.placeholder(tf.float32, shape=(None, 784))
input_layer = InputLayer(inputs)
outputs = Layer(input_layer, 10, self.session, non_liniarity=tf.sigmoid)
trainer = CategoricalTrainer(outputs, 0.1)
trainer.train(data.validation.images, data.validation.labels)
# this was throwing an exception
accuracy = trainer.accuracy(data.validation.images, data.validation.labels)
self.assertLessEqual(accuracy, 100.)
self.assertGreaterEqual(accuracy, 0.)
def test_mnist(self):
data = self.mnist_data
input = InputLayer(self.MNIST_INPUT_NODES)
d_1 = DuelStateReluLayer(input, 3, width_regularizer_constant=1e-7, width_binarizer_constant=1e-10,
session=self.session)
d_2 = DuelStateReluLayer(d_1, 3, width_regularizer_constant=1e-7, width_binarizer_constant=1e-10, )
output = HiddenLayer(d_2, self.MNIST_OUTPUT_NODES)
trainer = CategoricalTrainer(output, 0.1)
end_epoch = data.train.epochs_completed + 20
while data.train.epochs_completed <= end_epoch:
train_x, train_y = data.train.next_batch(100)
trainer.train(train_x, train_y)
accuracy, cost = trainer.accuracy(data.test.features, data.test.labels)
print(accuracy, cost)
print("active nodes ", d_1.active_nodes())
self.assertGreater(accuracy, 70.)
def test_mnist(self):
data = self.mnist_data
input = InputLayer(self.MNIST_INPUT_NODES)
d_1 = DuelStateReluLayer(input, 3, width_regularizer_constant=1e-7, width_binarizer_constant=1e-10,
session=self.session)
# when we add in batch norm layers we find that no active nodes are created, width is always less than 0.5?
# bn_1 = BatchNormLayer(d_1)
d_2 = DuelStateReluLayer(d_1, 3, width_regularizer_constant=1e-7, width_binarizer_constant=1e-10, )
# bn_2 = BatchNormLayer(d_2)
output = Layer(d_2, self.MNIST_OUTPUT_NODES)
trainer = CategoricalTrainer(output, 0.1)
end_epoch = data.train.epochs_completed + 20
while data.train.epochs_completed <= end_epoch:
train_x, train_y = data.train.next_batch(100)
trainer.train(train_x, train_y)
accuracy, cost = trainer.accuracy(data.test.images, data.test.labels)
print(accuracy, cost)
print("active nodes ", d_1.active_nodes())
self.assertGreater(accuracy, 70.)
bn4 = BatchNormLayer(net3, sess, beta=beta, gamma=gamma)
net4 = HiddenLayer(bn4,
1,
sess,
non_liniarity=non_lin,
bactivate=bactivate,
unsupervised_cost=.001,
noise_std=noise_std)
bn5 = BatchNormLayer(net4, sess, beta=beta, gamma=gamma)
outputNet = HiddenLayer(bn5,
10,
sess,
non_liniarity=tf.sigmoid,
bactivate=False,
supervised_cost=1.)
trainer = CategoricalTrainer(outputNet, 0.15)
trainPolicy = TrainPolicy(trainer,
data,
batch_size,
max_iterations=3000,
grow_after_turns_without_improvement=2,
start_grow_epoch=1,
learn_rate_decay=0.99,
learn_rate_boost=0.01,
back_loss_on_misclassified_only=True)
trainPolicy.run_full()
print trainer.accuracy(data.test.features, data.test.labels)
inputs = tf.placeholder(tf.float32, shape=(None, 784))
bactivate = True
noise_std = 0.3
beta = 0.5
gamma = 0.5
non_lin = tf.nn.sigmoid
input_layer = InputLayer(inputs)
bn1 = BatchNormLayer(input_layer, sess, beta=beta, gamma=gamma)
net1 = Layer(bn1, 1, sess, non_liniarity=non_lin, bactivate=bactivate, unsupervised_cost=.001, noise_std=noise_std)
bn2 = BatchNormLayer(net1, sess, beta=beta, gamma=gamma)
net2 = Layer(bn2, 1, sess, non_liniarity=non_lin, bactivate=bactivate, unsupervised_cost=.001, noise_std=noise_std)
bn3 = BatchNormLayer(net2, sess, beta=beta, gamma=gamma)
net3 = Layer(bn3, 1, sess, non_liniarity=non_lin, bactivate=bactivate, unsupervised_cost=.001, noise_std=noise_std)
bn4 = BatchNormLayer(net3, sess, beta=beta, gamma=gamma)
net4 = Layer(bn4, 1, sess, non_liniarity=non_lin, bactivate=bactivate, unsupervised_cost=.001, noise_std=noise_std)
bn5 = BatchNormLayer(net4, sess, beta=beta, gamma=gamma)
outputNet = Layer(bn5, 10, sess, non_liniarity=tf.sigmoid, bactivate=False, supervised_cost=1.)
trainer = CategoricalTrainer(outputNet, 0.15)
trainPolicy = TrainPolicy(trainer, data, batch_size, max_iterations=3000,
grow_after_turns_without_improvement=2,
start_grow_epoch=1,
learn_rate_decay=0.99,
learn_rate_boost=0.01,
back_loss_on_misclassified_only=True)
trainPolicy.run_full()
print trainer.accuracy(data.test.images, data.test.labels)
