def test_increase_layers_until_stop_decreasing_test_error(self):
data = self.mnist_data
input = InputLayer(784)
hidden = Layer(input, HIDDEN_NOES, self.session, non_liniarity=tf.sigmoid, bactivate=False)
output = Layer(hidden, 10, self.session, non_liniarity=tf.sigmoid, bactivate=False, supervised_cost=1.0)
best_score = train_until_no_improvement_for_epochs(data, output, 3)
for hidden_layer_count in range(1, 10):
print("hidden_layers {0} best_score {1}".format(hidden_layer_count, best_score))
candidate = output.clone()
last_hidden_layer = candidate.last_layer.input_layer
last_hidden_layer.add_intermediate_layer(
lambda input_layer: Layer(
input_layer, HIDDEN_NOES, self.session, non_liniarity=tf.sigmoid, bactivate=False
)
)
new_best_score = train_until_no_improvement_for_epochs(data, candidate, 3)
if new_best_score > best_score:
# failed to get improvement
print("failed to get improvement with layer {0}".format(hidden_layer_count))
break
else:
best_score = new_best_score
output = candidate
def test_get_layers_list(self):
input_p = tf.placeholder("float", (None, 10))
layer = Layer(InputLayer(input_p), 1, session=self.session)
layer2 = Layer(layer, 2, session=self.session)
layer3 = Layer(layer2, 3, session=self.session)
self.assertEquals(layer.get_layers_list(), [layer, layer2, layer3])
def test_layer_noisy_input_activation(self):
input_size = 100
noise_std = 1.
input_p = tf.placeholder("float", (None, input_size))
layer = Layer(InputLayer(input_p), input_size,
weights=np.diag(np.ones(input_size, dtype=np.float32)),
bias=np.zeros(input_size, dtype=np.float32),
session=self.session,
non_liniarity=tf.identity,
noise_std=noise_std)
result_noisy = self.session.run(layer.activation_train,
feed_dict={
input_p: np.ones(input_size, dtype=np.float32).reshape((1, input_size))})
self.assertAlmostEqual(result_noisy.std(), noise_std, delta=noise_std / 5.,
msg="the result std should be the noise_std")
layer.predict = True
result_clean = self.session.run(layer.activation_predict, feed_dict={
input_p: np.ones(input_size, dtype=np.float32).reshape((1, input_size))})
self.assertAlmostEqual(result_clean.std(), 0., places=7,
msg="There should be no noise in the activation")
def test_resize(self):
output_nodes = 10
input_p = tf.placeholder("float", (None, 10))
layer = Layer(InputLayer(input_p), output_nodes, session=self.session)
layer.resize(output_nodes + 1)
print layer._bias.get_shape()
self.assertEqual(layer.activation_predict.get_shape().as_list(), [None, output_nodes + 1])
self.assertEquals(layer.output_nodes, output_nodes + 1)
def test_resize(self):
inputs = tf.placeholder(tf.float32, shape=(None, 784))
bactivate = True
net1 = InputLayer(inputs)
net2 = Layer(net1, 10, self.session, bactivate=bactivate)
bn1 = BatchNormLayer(net2, self.session)
output_net = Layer(bn1, 10, self.session, bactivate=False)
print(self.session.run(output_net.activation_predict, feed_dict={inputs: np.zeros(shape=(1, 784))}))
net2.resize(net2.output_nodes + 1)
print(self.session.run(output_net.activation_predict, feed_dict={inputs: np.zeros(shape=(1, 784))}))
def test_reshape(self):
output_nodes = 2
input_p = tf.placeholder("float", (None, 2))
layer = Layer(InputLayer(input_p), output_nodes, session=self.session,
weights=np.array([[100.0]], dtype=np.float32))
result1 = self.session.run(layer.activation_predict, feed_dict={layer.input_placeholder: [[1., 1.]]})
layer.resize(3)
result2 = self.session.run(layer.activation_predict, feed_dict={layer.input_placeholder: [[1., 1.]]})
print(result1)
print(result2)
self.assertEquals(len(result2[0]), 3)
def test_reconstruction_of_single_input(self):
input_layer = InputLayer(1)
layer = Layer(input_layer, 1, bactivate=True, session=self.session, noise_std=0.3)
cost = layer.unsupervised_cost_train()
optimizer = tf.train.AdamOptimizer(0.1).minimize(cost)
self.session.run(tf.initialize_all_variables())
data = np.random.normal(0.5, 0.5, size=[200, 1])
for x in range(100):
self.session.run([optimizer], feed_dict={input_layer.input_placeholder: data})
result = self.session.run([cost], feed_dict={input_layer.input_placeholder: data})
print result
def test_clone(self):
inputs = tf.placeholder(tf.float32, shape=(None, 784))
net1 = InputLayer(inputs)
bn1 = BatchNormLayer(net1, self.session)
net2 = Layer(bn1, 8, self.session)
bn2 = BatchNormLayer(net2, self.session)
net2 = Layer(bn2, 6, self.session)
bn3 = BatchNormLayer(net2, self.session)
net3 = Layer(bn3, 4, self.session)
output_net = Layer(net3, 2, self.session)
cloned_net = output_net.clone(self.session)
self.assertNotEquals(cloned_net, output_net)
self.assertNotEquals(cloned_net.input_layer, output_net.input_layer)
self.assertEqual(len(list(cloned_net.all_layers)), len(list(output_net.all_layers)))
def reconstruction_loss_for(self, output_nodes):
data = self.mnist_data
bw_layer1 = Layer(InputLayer(784), output_nodes, session=self.session, noise_std=1.0, bactivate=True)
cost = bw_layer1.unsupervised_cost_train()
optimizer = tf.train.AdamOptimizer(0.1).minimize(cost)
self.session.run(tf.initialize_all_variables())
end_epoch = data.train.epochs_completed + 3
while data.train.epochs_completed <= end_epoch:
train_x, train_y = data.train.next_batch(100)
self.session.run(optimizer, feed_dict={bw_layer1.input_placeholder: train_x})
result = self.session.run(bw_layer1.unsupervised_cost_predict(),
feed_dict={bw_layer1.input_placeholder: data.test.images})
print("denoising with %s hidden layer had cost %s" % (output_nodes, result))
return result
def test_noise_reconstruction(self):
INPUT_DIM = 10
HIDDEN_NODES = 1
bw_layer1 = Layer(InputLayer(INPUT_DIM), HIDDEN_NODES, session=self.session, noise_std=1.0, bactivate=True)
# single cluster reconstruct
data = []
for i in range(10):
data.append([i*.1]*INPUT_DIM)
cost = bw_layer1.unsupervised_cost_train()
optimizer = tf.train.AdamOptimizer(0.1).minimize(cost)
self.session.run(tf.initialize_all_variables())
for j in range(200):
self.session.run(optimizer, feed_dict={bw_layer1.input_placeholder: data})
result = self.session.run(bw_layer1.unsupervised_cost_predict(),
feed_dict={bw_layer1.input_placeholder: data})
print("denoising with %s hidden layer had cost %s" % (HIDDEN_NODES, result))